[Model Runner V2][Spec Decode] Add multi-layer MTP speculator (#48892)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
@@ -282,6 +283,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
step: int,
|
||||
num_query_per_req: int | None = None,
|
||||
causal: bool | Mapping[int, bool] = False,
|
||||
query_start_loc_np: np.ndarray | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
if not self.draft_attn_layer_names:
|
||||
return None
|
||||
@@ -294,6 +296,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
step=step,
|
||||
num_query_per_req=self.num_query_per_req,
|
||||
causal=causal,
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
@@ -51,7 +52,7 @@ class BaseSpeculator(ABC):
|
||||
num_rejected: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
last_sampled: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
# [num_prefill_lookahead, max_num_reqs]
|
||||
next_prefill_tokens: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
temperature: torch.Tensor,
|
||||
@@ -214,14 +215,28 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
step: int,
|
||||
num_query_per_req: int = 1,
|
||||
causal: bool | Mapping[int, bool] = True,
|
||||
query_start_loc_np: np.ndarray | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
|
||||
# Clamp keeps the series non-decreasing past num_reqs, which some
|
||||
# attention backends require.
|
||||
query_start_loc_cpu = (
|
||||
torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs)
|
||||
* num_query_per_req
|
||||
)
|
||||
if query_start_loc_np is not None:
|
||||
# Non-uniform query layout (e.g. multi-module MTP's mixed
|
||||
# prefill/decode queries); num_query_per_req is ignored.
|
||||
query_start_loc_cpu = torch.empty(num_reqs_padded + 1, dtype=torch.int32)
|
||||
query_start_loc_cpu[: num_reqs + 1] = torch.from_numpy(
|
||||
query_start_loc_np[: num_reqs + 1]
|
||||
)
|
||||
query_start_loc_cpu[num_reqs:] = query_start_loc_cpu[num_reqs]
|
||||
max_query_len = int(
|
||||
(query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]).max()
|
||||
)
|
||||
else:
|
||||
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
|
||||
# Clamp keeps the series non-decreasing past num_reqs, which some
|
||||
# attention backends require.
|
||||
query_start_loc_cpu = (
|
||||
torch.clamp(self.arange[: num_reqs_padded + 1], max=num_reqs)
|
||||
* num_query_per_req
|
||||
)
|
||||
max_query_len = num_query_per_req
|
||||
block_tables = [
|
||||
x[:num_reqs_padded] for x in self.block_tables.input_block_tables
|
||||
]
|
||||
@@ -243,7 +258,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
: num_reqs_padded + 1
|
||||
],
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=num_query_per_req,
|
||||
max_query_len=max_query_len,
|
||||
seq_lens=self.input_buffers.seq_lens[:num_reqs_padded],
|
||||
max_seq_len=self.draft_max_seq_len,
|
||||
block_tables=block_tables,
|
||||
@@ -325,7 +340,6 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
self.temperature.copy_(temperature)
|
||||
self.seeds.copy_(seeds)
|
||||
self.idx_mapping[:num_reqs].copy_(idx_mapping)
|
||||
if self.draft_logits is not None:
|
||||
# idx_mapping for CG padded requests points to -1, which is ignored
|
||||
# during sampling to prevent writing stale values to draft logits.
|
||||
self.idx_mapping[num_reqs:].fill_(-1)
|
||||
# idx_mapping for CG padded requests points to -1, which is ignored
|
||||
# during sampling to prevent writing stale values to draft logits.
|
||||
self.idx_mapping[num_reqs:].fill_(-1)
|
||||
|
||||
Reference in New Issue
Block a user