[Bugfix] Index Mamba align state copies by request slot in MRV2 (#60210)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Karl0007 <44745209+Karl0007@users.noreply.github.com> Co-authored-by: Yongqin Wang <yongqinwang@roblox.com>
This commit is contained in:
co-authored by
Karl0007
Yongqin Wang
parent
956960f85f
commit
8a26869bca
@@ -778,6 +778,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.kv_caches = [
|
||||
cache for cache in kv_caches_dict.values() if cache.device == self.device
|
||||
]
|
||||
self.model_state.initialize_kv_cache(self.kv_cache_config, self.block_tables)
|
||||
if is_profiling:
|
||||
self.kv_connector = NO_OP_KV_CONNECTOR
|
||||
else:
|
||||
|
||||
@@ -127,6 +127,12 @@ class ModelState(ABC):
|
||||
def apply_staged_writes(self) -> None:
|
||||
return None
|
||||
|
||||
def initialize_kv_cache(
|
||||
self, kv_cache_config: KVCacheConfig, block_tables: BlockTables
|
||||
) -> None:
|
||||
"""Hook run after the KV cache tensors are allocated and bound."""
|
||||
return None
|
||||
|
||||
def capture_inner_cudagraphs(
|
||||
self,
|
||||
input_buffers: InputBuffers,
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.v1.core.sched.output import NewRequestData
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.utils import CpuGpuBuffer
|
||||
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
@@ -134,47 +135,42 @@ class MambaHybridModelState(DefaultModelState):
|
||||
self._mamba_spec = mamba_spec
|
||||
return self._mamba_group_ids, self._mamba_spec
|
||||
|
||||
def _ensure_align_ctx(
|
||||
self,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
mamba_group_ids: list[int],
|
||||
block_tables: tuple[torch.Tensor, ...],
|
||||
) -> MambaSpecDecodeGPUContext:
|
||||
def initialize_kv_cache(
|
||||
self, kv_cache_config: KVCacheConfig, block_tables: BlockTables
|
||||
) -> None:
|
||||
if not self._align_mode:
|
||||
return
|
||||
mamba_group_ids, _ = self._get_mamba_group_info(kv_cache_config)
|
||||
if self._mamba_state_copy_funcs is None:
|
||||
mamba_groups = get_mamba_groups(kv_cache_config)
|
||||
mamba_types = {spec.mamba_type for spec in mamba_groups}
|
||||
copy_funcs = self.model.get_mamba_state_copy_funcs(mamba_types)
|
||||
validate_mamba_state_copy_funcs(mamba_groups, copy_funcs)
|
||||
self._mamba_state_copy_funcs = copy_funcs
|
||||
copy_funcs = self._mamba_state_copy_funcs
|
||||
if self._mamba_ctx is None:
|
||||
# Both SD and DS conv layouts support a >0 spec-decode shift: the
|
||||
# fused pre-copy kernel (``_copy_mamba_state_block``) applies the
|
||||
# ``token_bias = num_accepted - 1`` window shift per conv layout
|
||||
# (SD: contiguous slice; DS: per-dim-row strided slice), matching
|
||||
# the V1 ``get_conv_copy_spec`` semantics.
|
||||
self._mamba_ctx = MambaSpecDecodeGPUContext.create(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
kv_cache_config=kv_cache_config,
|
||||
copy_funcs=copy_funcs,
|
||||
device=self.device,
|
||||
make_buffer=lambda n, dtype: CpuGpuBuffer(
|
||||
n, dtype=dtype, device=self.device
|
||||
),
|
||||
)
|
||||
ctx = self._mamba_ctx
|
||||
if not ctx.is_initialized:
|
||||
forward_context = self.vllm_config.compilation_config.static_forward_context
|
||||
# block_tables are batch-order slices of the persistent
|
||||
# input_block_tables (stable data_ptr), so the metadata is captured
|
||||
# once here and reused across steps.
|
||||
ctx.initialize_from_forward_context(
|
||||
kv_cache_config,
|
||||
forward_context,
|
||||
copy_funcs,
|
||||
[block_tables[gid] for gid in mamba_group_ids],
|
||||
)
|
||||
return ctx
|
||||
# Both SD and DS conv layouts support a >0 spec-decode shift: the
|
||||
# fused pre-copy kernel (``_copy_mamba_state_block``) applies the
|
||||
# ``token_bias = num_accepted - 1`` window shift per conv layout
|
||||
# (SD: contiguous slice; DS: per-dim-row strided slice), matching
|
||||
# the V1 ``get_conv_copy_spec`` semantics.
|
||||
self._mamba_ctx = MambaSpecDecodeGPUContext.create(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
kv_cache_config=kv_cache_config,
|
||||
copy_funcs=self._mamba_state_copy_funcs,
|
||||
device=self.device,
|
||||
make_buffer=lambda n, dtype: CpuGpuBuffer(
|
||||
n, dtype=dtype, device=self.device
|
||||
),
|
||||
)
|
||||
# Under PP a step's postprocess runs after later steps re-gathered the
|
||||
# batch-ordered input tables, so state copies read the per-request-slot
|
||||
# tables. Aligned state indices are for the current batch only.
|
||||
self._mamba_ctx.initialize_from_forward_context(
|
||||
kv_cache_config,
|
||||
self.vllm_config.compilation_config.static_forward_context,
|
||||
self._mamba_state_copy_funcs,
|
||||
[block_tables.block_tables[gid].gpu for gid in mamba_group_ids],
|
||||
[block_tables.input_block_tables[gid] for gid in mamba_group_ids],
|
||||
)
|
||||
|
||||
def preprocess_state(
|
||||
self,
|
||||
@@ -194,8 +190,9 @@ class MambaHybridModelState(DefaultModelState):
|
||||
num_reqs = input_batch.num_reqs
|
||||
if num_reqs == 0:
|
||||
return
|
||||
mamba_group_ids, mamba_spec = self._get_mamba_group_info(kv_cache_config)
|
||||
ctx = self._ensure_align_ctx(kv_cache_config, mamba_group_ids, block_tables)
|
||||
_, mamba_spec = self._get_mamba_group_info(kv_cache_config)
|
||||
ctx = self._mamba_ctx
|
||||
assert ctx is not None
|
||||
|
||||
# The state-advance + pre-copy kernels run every step; they fast-exit per
|
||||
# request when src_col < 0 or src_col == dst_col, so no copy happens on
|
||||
@@ -301,10 +298,8 @@ class MambaHybridModelState(DefaultModelState):
|
||||
if hasattr(builder, "mamba_aligned_state_indices"):
|
||||
aligned_index_builders.append((group_idx, builder))
|
||||
if aligned_index_builders:
|
||||
ctx = self._ensure_align_ctx(
|
||||
kv_cache_config, mamba_group_ids, block_tables
|
||||
)
|
||||
all_group_indices = ctx.compute_aligned_state_indices(
|
||||
assert self._mamba_ctx is not None
|
||||
all_group_indices = self._mamba_ctx.compute_aligned_state_indices(
|
||||
input_batch.seq_lens, num_reqs
|
||||
)
|
||||
for group_idx, builder in aligned_index_builders:
|
||||
|
||||
Reference in New Issue
Block a user