[Model Runner V2][Spec Decode] Add multi-layer MTP speculator (#48892)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -191,6 +191,8 @@ class InputBatch:
|
||||
def _prepare_prefill_inputs_kernel(
|
||||
input_ids_ptr,
|
||||
next_prefill_tokens_ptr,
|
||||
next_prefill_tokens_stride,
|
||||
num_lookahead,
|
||||
idx_mapping_ptr,
|
||||
query_start_loc_ptr,
|
||||
all_token_ids_ptr,
|
||||
@@ -198,6 +200,7 @@ def _prepare_prefill_inputs_kernel(
|
||||
prefill_lens_ptr,
|
||||
num_computed_tokens_ptr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
LOOKAHEAD_BLOCK: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + batch_idx)
|
||||
@@ -218,10 +221,20 @@ def _prepare_prefill_inputs_kernel(
|
||||
tokens = tl.load(request_ptr + num_computed + block, mask=mask)
|
||||
tl.store(input_ids_ptr + query_start + block, tokens, mask=mask)
|
||||
|
||||
next_pos = num_computed + query_len
|
||||
if next_pos < prefill_len:
|
||||
next_token = tl.load(request_ptr + next_pos)
|
||||
tl.store(next_prefill_tokens_ptr + req_state_idx, next_token)
|
||||
# Store the next num_lookahead prefill tokens.
|
||||
lookahead = tl.arange(0, LOOKAHEAD_BLOCK)
|
||||
pos = num_computed + query_len + lookahead
|
||||
in_lookahead = lookahead < num_lookahead
|
||||
tokens = tl.load(
|
||||
request_ptr + pos, mask=in_lookahead & (pos < prefill_len), other=0
|
||||
)
|
||||
tl.store(
|
||||
next_prefill_tokens_ptr
|
||||
+ lookahead * next_prefill_tokens_stride
|
||||
+ req_state_idx,
|
||||
tokens,
|
||||
mask=in_lookahead,
|
||||
)
|
||||
|
||||
|
||||
def prepare_prefill_inputs(
|
||||
@@ -234,9 +247,12 @@ def prepare_prefill_inputs(
|
||||
num_computed_tokens: torch.Tensor,
|
||||
) -> None:
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
num_lookahead = next_prefill_tokens.shape[0]
|
||||
_prepare_prefill_inputs_kernel[(num_reqs,)](
|
||||
input_ids,
|
||||
next_prefill_tokens,
|
||||
next_prefill_tokens.stride(0),
|
||||
num_lookahead,
|
||||
idx_mapping,
|
||||
query_start_loc,
|
||||
all_token_ids,
|
||||
@@ -244,6 +260,7 @@ def prepare_prefill_inputs(
|
||||
prefill_len,
|
||||
num_computed_tokens,
|
||||
BLOCK_SIZE=1024,
|
||||
LOOKAHEAD_BLOCK=triton.next_power_of_2(num_lookahead),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user