[Core][KDA] Generalize Mamba prefill checkpoint builder and exporter (#57783)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
@@ -0,0 +1,137 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
MambaSpec,
|
||||
get_mamba_prefill_checkpoint_position,
|
||||
is_mamba_prefill_checkpoint_valid,
|
||||
)
|
||||
|
||||
|
||||
def compute_mamba_prefill_checkpoints(
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
hash_block_size: int,
|
||||
mamba_block_size: int,
|
||||
checkpoint_alignment: int | None,
|
||||
drop_eagle_block: bool,
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""Per-row internal prefill checkpoint offsets and cache block columns.
|
||||
|
||||
Backends call this instead of re-deriving the rules, so they decline in
|
||||
lockstep with the scheduler and ``MambaManager``: allocating a checkpoint
|
||||
block without writing it leaves the prefix cache serving uninitialized
|
||||
state.
|
||||
|
||||
Returns:
|
||||
``(offsets, cols)``: the checkpoint's offset into each row's query and
|
||||
its block-table column. ``0`` and ``-1`` mean the row has none.
|
||||
|
||||
"""
|
||||
offsets: list[int] = []
|
||||
cols: list[int] = []
|
||||
for seq_len, query_len in zip(seq_lens, query_lens):
|
||||
query_start = seq_len - query_len
|
||||
position = get_mamba_prefill_checkpoint_position(
|
||||
seq_len, hash_block_size, drop_eagle_block=drop_eagle_block
|
||||
)
|
||||
valid = is_mamba_prefill_checkpoint_valid(
|
||||
query_start=query_start,
|
||||
query_end=seq_len,
|
||||
checkpoint_position=position,
|
||||
hash_block_size=hash_block_size,
|
||||
mamba_block_size=mamba_block_size,
|
||||
checkpoint_alignment=checkpoint_alignment,
|
||||
)
|
||||
offsets.append(position - query_start if valid else 0)
|
||||
cols.append(cdiv(seq_len, mamba_block_size) - 2 if valid else -1)
|
||||
return offsets, cols
|
||||
|
||||
|
||||
@dataclass
|
||||
class MambaPrefillCheckpointMetadata:
|
||||
checkpoint_offsets: torch.Tensor
|
||||
state_indices: torch.Tensor
|
||||
|
||||
|
||||
class MambaPrefillCheckpointBuilder:
|
||||
"""Build backend-neutral prefill checkpoint locations."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, kv_cache_spec: MambaSpec) -> None:
|
||||
self.vllm_config = vllm_config
|
||||
self.kv_cache_spec = kv_cache_spec
|
||||
|
||||
def build(
|
||||
self,
|
||||
m: CommonAttentionMetadata,
|
||||
request_rows: list[int],
|
||||
) -> MambaPrefillCheckpointMetadata | None:
|
||||
if self.kv_cache_spec.num_prefill_checkpoint_blocks == 0:
|
||||
return None
|
||||
assert m.seq_lens_cpu_upper_bound is not None
|
||||
all_query_lens = m.query_start_loc_cpu.diff().tolist()
|
||||
query_lens = [all_query_lens[row] for row in request_rows]
|
||||
seq_lens = m.seq_lens_cpu_upper_bound.tolist()
|
||||
block_size = self.kv_cache_spec.block_size
|
||||
hash_block_size = self.vllm_config.cache_config.prefix_match_unit or block_size
|
||||
speculative_config = self.vllm_config.speculative_config
|
||||
drop_eagle_block = (
|
||||
speculative_config is not None and speculative_config.use_eagle_block_drop()
|
||||
)
|
||||
checkpoint_offsets, checkpoint_cols = compute_mamba_prefill_checkpoints(
|
||||
[seq_lens[row] for row in request_rows],
|
||||
query_lens,
|
||||
hash_block_size=hash_block_size,
|
||||
mamba_block_size=block_size,
|
||||
checkpoint_alignment=self.kv_cache_spec.prefill_checkpoint_alignment,
|
||||
drop_eagle_block=drop_eagle_block,
|
||||
)
|
||||
if not any(checkpoint_offsets):
|
||||
return None
|
||||
checkpoint_offsets_tensor = async_tensor_h2d(
|
||||
checkpoint_offsets,
|
||||
dtype=torch.int32,
|
||||
device=m.query_start_loc.device,
|
||||
)
|
||||
request_rows_tensor = async_tensor_h2d(
|
||||
request_rows, dtype=torch.int64, device=m.query_start_loc.device
|
||||
)
|
||||
checkpoint_cols_tensor = async_tensor_h2d(
|
||||
checkpoint_cols, dtype=torch.int64, device=m.query_start_loc.device
|
||||
)
|
||||
checkpoint_state_indices = m.block_table_tensor[
|
||||
request_rows_tensor, checkpoint_cols_tensor
|
||||
]
|
||||
checkpoint_state_indices = torch.where(
|
||||
checkpoint_cols_tensor >= 0,
|
||||
checkpoint_state_indices,
|
||||
NULL_BLOCK_ID,
|
||||
)
|
||||
return MambaPrefillCheckpointMetadata(
|
||||
checkpoint_offsets_tensor,
|
||||
checkpoint_state_indices,
|
||||
)
|
||||
|
||||
|
||||
class MambaPrefillCheckpointExporter(ABC):
|
||||
"""Export a mid-prefill checkpoint into backend-specific paged states."""
|
||||
|
||||
@abstractmethod
|
||||
def export(
|
||||
self,
|
||||
checkpoint: MambaPrefillCheckpointMetadata,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""Write checkpoint state into the paged cache."""
|
||||
raise NotImplementedError
|
||||
@@ -0,0 +1,130 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointExporter,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
def kda_prefill_checkpoint_alignment(backend: str) -> int | None:
|
||||
return 16 if backend == "flashkda" else None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FlashKDAPrefillCheckpointExporter(MambaPrefillCheckpointExporter):
|
||||
"""Store FlashKDA recurrent and convolution checkpoint states."""
|
||||
|
||||
state_len: int | None = None
|
||||
|
||||
def export(
|
||||
self,
|
||||
checkpoint: MambaPrefillCheckpointMetadata,
|
||||
*,
|
||||
raw_qkv: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
recurrent_checkpoint: torch.Tensor,
|
||||
recurrent_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
) -> None:
|
||||
state_len = (
|
||||
self.state_len if self.state_len is not None else conv_state.shape[-1]
|
||||
)
|
||||
width = raw_qkv.shape[-1]
|
||||
recurrent_row_size = recurrent_checkpoint[0].numel()
|
||||
block_size = 256
|
||||
store_cache_checkpoints_kernel[
|
||||
(
|
||||
checkpoint.checkpoint_offsets.numel(),
|
||||
triton.cdiv(max(width * state_len, recurrent_row_size), block_size),
|
||||
)
|
||||
](
|
||||
raw_qkv,
|
||||
conv_state,
|
||||
recurrent_checkpoint,
|
||||
recurrent_state,
|
||||
cu_seqlens,
|
||||
checkpoint.checkpoint_offsets,
|
||||
checkpoint.state_indices,
|
||||
raw_qkv.stride(0),
|
||||
raw_qkv.stride(1),
|
||||
conv_state.stride(0),
|
||||
conv_state.stride(1),
|
||||
conv_state.stride(2),
|
||||
recurrent_checkpoint.stride(0),
|
||||
recurrent_state.stride(0),
|
||||
checkpoint.checkpoint_offsets.stride(0),
|
||||
state_len,
|
||||
width,
|
||||
recurrent_row_size,
|
||||
NULL_BLOCK_ID,
|
||||
block_size,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def store_cache_checkpoints_kernel(
|
||||
x_ptr,
|
||||
conv_state_ptr,
|
||||
recurrent_checkpoint_ptr,
|
||||
recurrent_state_ptr,
|
||||
query_start_loc_ptr,
|
||||
checkpoint_offsets_ptr,
|
||||
checkpoint_state_indices_ptr,
|
||||
x_stride_0: tl.constexpr,
|
||||
x_stride_1: tl.constexpr,
|
||||
state_stride_0: tl.constexpr,
|
||||
state_stride_1: tl.constexpr,
|
||||
state_stride_2: tl.constexpr,
|
||||
checkpoint_stride_0: tl.constexpr,
|
||||
recurrent_state_stride_0: tl.constexpr,
|
||||
checkpoint_offset_stride: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
RECURRENT_ROW_SIZE: tl.constexpr,
|
||||
NULL_STATE_IDX: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
seq_idx = tl.program_id(0)
|
||||
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
state_idx = tl.load(checkpoint_state_indices_ptr + seq_idx).to(tl.int64)
|
||||
checkpoint_offset = tl.load(
|
||||
checkpoint_offsets_ptr + seq_idx * checkpoint_offset_stride
|
||||
)
|
||||
valid_checkpoint = (state_idx != NULL_STATE_IDX) & (checkpoint_offset > 0)
|
||||
valid_conv = (
|
||||
(cols < WIDTH * STATE_LEN) & valid_checkpoint & (checkpoint_offset >= STATE_LEN)
|
||||
)
|
||||
width_idx = cols // STATE_LEN
|
||||
history_idx = cols % STATE_LEN
|
||||
checkpoint_end = tl.load(query_start_loc_ptr + seq_idx) + checkpoint_offset
|
||||
token_idx = checkpoint_end - STATE_LEN + history_idx
|
||||
values = tl.load(
|
||||
x_ptr + token_idx * x_stride_0 + width_idx * x_stride_1,
|
||||
mask=valid_conv,
|
||||
)
|
||||
tl.store(
|
||||
conv_state_ptr
|
||||
+ state_idx * state_stride_0
|
||||
+ width_idx * state_stride_1
|
||||
+ history_idx * state_stride_2,
|
||||
values,
|
||||
mask=valid_conv,
|
||||
)
|
||||
|
||||
valid_recurrent = (cols < RECURRENT_ROW_SIZE) & valid_checkpoint
|
||||
recurrent = tl.load(
|
||||
recurrent_checkpoint_ptr + seq_idx * checkpoint_stride_0 + cols,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
tl.store(
|
||||
recurrent_state_ptr + state_idx * recurrent_state_stride_0 + cols,
|
||||
recurrent,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
Reference in New Issue
Block a user