[Bugfix][MRV2] Keep GDN prefill checkpoint metadata local to each cache group (#59536)
Signed-off-by: Jingqiao Zhang <ai-jz@users.noreply.github.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Jingqiao Zhang <ai-jz@users.noreply.github.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Jingqiao Zhang
zjy0516
parent
5d9214d6b2
commit
b538d807ff
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import torch
|
||||
|
||||
@@ -58,10 +58,33 @@ def compute_mamba_prefill_checkpoints(
|
||||
return offsets, cols
|
||||
|
||||
|
||||
def _gather_checkpoint_state_indices(
|
||||
block_table: torch.Tensor, request_rows: torch.Tensor, block_cols: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return torch.where(
|
||||
block_cols >= 0, block_table[request_rows, block_cols], NULL_BLOCK_ID
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MambaPrefillCheckpointMetadata:
|
||||
checkpoint_offsets: torch.Tensor
|
||||
state_indices: torch.Tensor
|
||||
# Gather indices into the block table, kept so that another KV cache group
|
||||
# with the same spec can re-derive its own ``state_indices``.
|
||||
request_rows: torch.Tensor | None = None
|
||||
block_cols: torch.Tensor | None = None
|
||||
|
||||
def regather_state_indices(
|
||||
self, block_table: torch.Tensor
|
||||
) -> "MambaPrefillCheckpointMetadata":
|
||||
assert self.request_rows is not None and self.block_cols is not None
|
||||
return replace(
|
||||
self,
|
||||
state_indices=_gather_checkpoint_state_indices(
|
||||
block_table, self.request_rows, self.block_cols
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class MambaPrefillCheckpointBuilder:
|
||||
@@ -111,17 +134,13 @@ class MambaPrefillCheckpointBuilder:
|
||||
checkpoint_cols_tensor = async_tensor_h2d(
|
||||
checkpoint_cols, dtype=torch.int64, device=m.query_start_loc.device
|
||||
)
|
||||
checkpoint_state_indices = m.block_table_tensor[
|
||||
request_rows_tensor, checkpoint_cols_tensor
|
||||
]
|
||||
checkpoint_state_indices = torch.where(
|
||||
checkpoint_cols_tensor >= 0,
|
||||
checkpoint_state_indices,
|
||||
NULL_BLOCK_ID,
|
||||
)
|
||||
return MambaPrefillCheckpointMetadata(
|
||||
checkpoint_offsets_tensor,
|
||||
checkpoint_state_indices,
|
||||
_gather_checkpoint_state_indices(
|
||||
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
|
||||
),
|
||||
request_rows=request_rows_tensor,
|
||||
block_cols=checkpoint_cols_tensor,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user