[Model Runner V2][Spec Decode] Add multi-layer MTP speculator (#48892)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -191,6 +191,8 @@ class InputBatch:
|
||||
def _prepare_prefill_inputs_kernel(
|
||||
input_ids_ptr,
|
||||
next_prefill_tokens_ptr,
|
||||
next_prefill_tokens_stride,
|
||||
num_lookahead,
|
||||
idx_mapping_ptr,
|
||||
query_start_loc_ptr,
|
||||
all_token_ids_ptr,
|
||||
@@ -198,6 +200,7 @@ def _prepare_prefill_inputs_kernel(
|
||||
prefill_lens_ptr,
|
||||
num_computed_tokens_ptr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
LOOKAHEAD_BLOCK: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + batch_idx)
|
||||
@@ -218,10 +221,20 @@ def _prepare_prefill_inputs_kernel(
|
||||
tokens = tl.load(request_ptr + num_computed + block, mask=mask)
|
||||
tl.store(input_ids_ptr + query_start + block, tokens, mask=mask)
|
||||
|
||||
next_pos = num_computed + query_len
|
||||
if next_pos < prefill_len:
|
||||
next_token = tl.load(request_ptr + next_pos)
|
||||
tl.store(next_prefill_tokens_ptr + req_state_idx, next_token)
|
||||
# Store the next num_lookahead prefill tokens.
|
||||
lookahead = tl.arange(0, LOOKAHEAD_BLOCK)
|
||||
pos = num_computed + query_len + lookahead
|
||||
in_lookahead = lookahead < num_lookahead
|
||||
tokens = tl.load(
|
||||
request_ptr + pos, mask=in_lookahead & (pos < prefill_len), other=0
|
||||
)
|
||||
tl.store(
|
||||
next_prefill_tokens_ptr
|
||||
+ lookahead * next_prefill_tokens_stride
|
||||
+ req_state_idx,
|
||||
tokens,
|
||||
mask=in_lookahead,
|
||||
)
|
||||
|
||||
|
||||
def prepare_prefill_inputs(
|
||||
@@ -234,9 +247,12 @@ def prepare_prefill_inputs(
|
||||
num_computed_tokens: torch.Tensor,
|
||||
) -> None:
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
num_lookahead = next_prefill_tokens.shape[0]
|
||||
_prepare_prefill_inputs_kernel[(num_reqs,)](
|
||||
input_ids,
|
||||
next_prefill_tokens,
|
||||
next_prefill_tokens.stride(0),
|
||||
num_lookahead,
|
||||
idx_mapping,
|
||||
query_start_loc,
|
||||
all_token_ids,
|
||||
@@ -244,6 +260,7 @@ def prepare_prefill_inputs(
|
||||
prefill_len,
|
||||
num_computed_tokens,
|
||||
BLOCK_SIZE=1024,
|
||||
LOOKAHEAD_BLOCK=triton.next_power_of_2(num_lookahead),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -223,6 +223,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.is_pooling_model = self.model_config.runner_type == "pooling"
|
||||
self.pooling_runner: PoolingRunner | None = None
|
||||
|
||||
# Multi-module MTP feeds its modules the next num_speculative_steps prefill
|
||||
# tokens during chunked prefill. Other speculators only read the immediate
|
||||
# next one.
|
||||
num_prefill_lookahead = (
|
||||
self.num_speculative_steps
|
||||
if self.speculative_config is not None
|
||||
and self.speculative_config.use_multi_module_mtp()
|
||||
else 1
|
||||
)
|
||||
# General request states.
|
||||
self.req_states = RequestState(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
@@ -231,6 +240,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_speculative_steps=self.num_speculative_steps,
|
||||
vocab_size=self.vocab_size,
|
||||
device=self.device,
|
||||
num_prefill_lookahead=num_prefill_lookahead,
|
||||
)
|
||||
self.input_buffers = InputBuffers(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
@@ -628,7 +638,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
torch.zeros(
|
||||
input_batch.num_tokens,
|
||||
dtype=torch.bool,
|
||||
device=self.device,
|
||||
device="cpu",
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1521,6 +1531,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# NOTE: This is done here because postprocess updates
|
||||
# num_computed_prefill_tokens.
|
||||
# The EAGLE/MTP drafter reads one position ahead of the target.
|
||||
# TODO(TheEpicDolphin): Gather MM embeddings for all speculative
|
||||
# steps during multi-module MTP.
|
||||
mm_inputs = self.model_state.gather_mm_embeddings(
|
||||
input_batch, draft_lookahead=1
|
||||
)
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
@@ -282,6 +283,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
step: int,
|
||||
num_query_per_req: int | None = None,
|
||||
causal: bool | Mapping[int, bool] = False,
|
||||
query_start_loc_np: np.ndarray | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
if not self.draft_attn_layer_names:
|
||||
return None
|
||||
@@ -294,6 +296,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
step=step,
|
||||
num_query_per_req=self.num_query_per_req,
|
||||
causal=causal,
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
@@ -51,7 +52,7 @@ class BaseSpeculator(ABC):
|
||||
num_rejected: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
last_sampled: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
# [num_prefill_lookahead, max_num_reqs]
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
temperature: torch.Tensor,
|
||||
@@ -214,14 +215,28 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
step: int,
|
||||
num_query_per_req: int = 1,
|
||||
causal: bool | Mapping[int, bool] = True,
|
||||
query_start_loc_np: np.ndarray | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
|
||||
# Clamp keeps the series non-decreasing past num_reqs, which some
|
||||
# attention backends require.
|
||||
query_start_loc_cpu = (
|
||||
torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs)
|
||||
* num_query_per_req
|
||||
)
|
||||
if query_start_loc_np is not None:
|
||||
# Non-uniform query layout (e.g. multi-module MTP's mixed
|
||||
# prefill/decode queries); num_query_per_req is ignored.
|
||||
query_start_loc_cpu = torch.empty(num_reqs_padded + 1, dtype=torch.int32)
|
||||
query_start_loc_cpu[: num_reqs + 1] = torch.from_numpy(
|
||||
query_start_loc_np[: num_reqs + 1]
|
||||
)
|
||||
query_start_loc_cpu[num_reqs:] = query_start_loc_cpu[num_reqs]
|
||||
max_query_len = int(
|
||||
(query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]).max()
|
||||
)
|
||||
else:
|
||||
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
|
||||
# Clamp keeps the series non-decreasing past num_reqs, which some
|
||||
# attention backends require.
|
||||
query_start_loc_cpu = (
|
||||
torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs)
|
||||
* num_query_per_req
|
||||
)
|
||||
max_query_len = num_query_per_req
|
||||
block_tables = [
|
||||
x[:num_reqs_padded] for x in self.block_tables.input_block_tables
|
||||
]
|
||||
@@ -243,7 +258,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
: num_reqs_padded + 1
|
||||
],
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=num_query_per_req,
|
||||
max_query_len=max_query_len,
|
||||
seq_lens=self.input_buffers.seq_lens[:num_reqs_padded],
|
||||
max_seq_len=self.draft_max_seq_len,
|
||||
block_tables=block_tables,
|
||||
@@ -325,7 +340,6 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
self.temperature.copy_(temperature)
|
||||
self.seeds.copy_(seeds)
|
||||
self.idx_mapping[:num_reqs].copy_(idx_mapping)
|
||||
if self.draft_logits is not None:
|
||||
# idx_mapping for CG padded requests points to -1, which is ignored
|
||||
# during sampling to prevent writing stale values to draft logits.
|
||||
self.idx_mapping[num_reqs:].fill_(-1)
|
||||
# idx_mapping for CG padded requests points to -1, which is ignored
|
||||
# during sampling to prevent writing stale values to draft logits.
|
||||
self.idx_mapping[num_reqs:].fill_(-1)
|
||||
|
||||
@@ -15,6 +15,7 @@ class RequestState:
|
||||
num_speculative_steps: int,
|
||||
vocab_size: int,
|
||||
device: torch.device,
|
||||
num_prefill_lookahead: int = 1,
|
||||
):
|
||||
self.max_num_reqs = max_num_reqs
|
||||
self.max_model_len = max_model_len
|
||||
@@ -77,7 +78,10 @@ class RequestState:
|
||||
)
|
||||
|
||||
self.next_prefill_tokens = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int32, device=device
|
||||
num_prefill_lookahead,
|
||||
self.max_num_reqs,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
Reference in New Issue
Block a user