[Bugfix][MRV2] Keep GDN prefill checkpoint metadata local to each cache group (#59536)

Signed-off-by: Jingqiao Zhang <ai-jz@users.noreply.github.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Jingqiao Zhang <ai-jz@users.noreply.github.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
ai-jz
2026-10-01 22:59:15 +08:00
committed by GitHub
co-authored by Jingqiao Zhang zjy0516
parent 5d9214d6b2
commit b538d807ff
3 changed files with 83 additions and 18 deletions
+29 -10
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from dataclasses import dataclass
from dataclasses import dataclass, replace
import torch
@@ -58,10 +58,33 @@ def compute_mamba_prefill_checkpoints(
return offsets, cols
def _gather_checkpoint_state_indices(
block_table: torch.Tensor, request_rows: torch.Tensor, block_cols: torch.Tensor
) -> torch.Tensor:
return torch.where(
block_cols >= 0, block_table[request_rows, block_cols], NULL_BLOCK_ID
)
@dataclass
class MambaPrefillCheckpointMetadata:
checkpoint_offsets: torch.Tensor
state_indices: torch.Tensor
# Gather indices into the block table, kept so that another KV cache group
# with the same spec can re-derive its own ``state_indices``.
request_rows: torch.Tensor | None = None
block_cols: torch.Tensor | None = None
def regather_state_indices(
self, block_table: torch.Tensor
) -> "MambaPrefillCheckpointMetadata":
assert self.request_rows is not None and self.block_cols is not None
return replace(
self,
state_indices=_gather_checkpoint_state_indices(
block_table, self.request_rows, self.block_cols
),
)
class MambaPrefillCheckpointBuilder:
@@ -111,17 +134,13 @@ class MambaPrefillCheckpointBuilder:
checkpoint_cols_tensor = async_tensor_h2d(
checkpoint_cols, dtype=torch.int64, device=m.query_start_loc.device
)
checkpoint_state_indices = m.block_table_tensor[
request_rows_tensor, checkpoint_cols_tensor
]
checkpoint_state_indices = torch.where(
checkpoint_cols_tensor >= 0,
checkpoint_state_indices,
NULL_BLOCK_ID,
)
return MambaPrefillCheckpointMetadata(
checkpoint_offsets_tensor,
checkpoint_state_indices,
_gather_checkpoint_state_indices(
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
),
request_rows=request_rows_tensor,
block_cols=checkpoint_cols_tensor,
)