[BUGFIX][Mamba][Qwen3.5] Zero freed SSM cache blocks on GPU (#35219)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
This commit is contained in:
@@ -197,6 +197,7 @@ from vllm.v1.worker.workspace import lock_workspace
|
||||
|
||||
from .utils import (
|
||||
AttentionGroup,
|
||||
KVBlockZeroer,
|
||||
add_kv_sharing_layers_to_kv_cache_groups,
|
||||
bind_kv_cache,
|
||||
prepare_kernel_block_sizes,
|
||||
@@ -982,6 +983,26 @@ class GPUModelRunner(
|
||||
decode_threshold=self.reorder_batch_threshold,
|
||||
)
|
||||
|
||||
def _init_kv_zero_meta(self) -> None:
|
||||
"""One-time precomputation for _zero_block_ids.
|
||||
|
||||
Delegates to KVBlockZeroer.init_meta with the runner's state.
|
||||
Called from gpu_worker.py outside the CuMem pool context.
|
||||
"""
|
||||
self._kv_block_zeroer = KVBlockZeroer(self.device, self.pin_memory)
|
||||
self._kv_block_zeroer.init_meta(
|
||||
attn_groups_iter=self._kv_cache_spec_attn_group_iterator(),
|
||||
kernel_block_sizes=self._kernel_block_sizes,
|
||||
cache_dtype=self.cache_config.cache_dtype,
|
||||
runner_only_attn_layers=self.runner_only_attn_layers,
|
||||
static_forward_context=(self.compilation_config.static_forward_context),
|
||||
)
|
||||
|
||||
def _zero_block_ids(self, block_ids: list[int]) -> None:
|
||||
"""Zero the KV cache memory for the given block IDs."""
|
||||
if hasattr(self, "_kv_block_zeroer"):
|
||||
self._kv_block_zeroer.zero_block_ids(block_ids)
|
||||
|
||||
# Note: used for model runner override.
|
||||
def _init_device_properties(self) -> None:
|
||||
"""Initialize attributes from torch.cuda.get_device_properties"""
|
||||
@@ -1018,6 +1039,11 @@ class GPUModelRunner(
|
||||
for req_id in scheduler_output.finished_req_ids:
|
||||
self.input_batch.remove_request(req_id)
|
||||
|
||||
# Zero GPU memory for freshly allocated cache blocks to prevent
|
||||
# stale NaN/data from corrupting attention or SSM computation.
|
||||
if scheduler_output.new_block_ids_to_zero:
|
||||
self._zero_block_ids(scheduler_output.new_block_ids_to_zero)
|
||||
|
||||
# Free the cached encoder outputs.
|
||||
for mm_hash in scheduler_output.free_encoder_mm_hashes:
|
||||
self.encoder_cache.pop(mm_hash, None)
|
||||
@@ -6476,6 +6502,7 @@ class GPUModelRunner(
|
||||
kernel_block_sizes = prepare_kernel_block_sizes(
|
||||
kv_cache_config, self.attn_groups
|
||||
)
|
||||
self._kernel_block_sizes = kernel_block_sizes
|
||||
|
||||
# create metadata builders
|
||||
self.initialize_metadata_builders(kv_cache_config, kernel_block_sizes)
|
||||
|
||||
Reference in New Issue
Block a user