[BUGFIX][Mamba][Qwen3.5] Zero freed SSM cache blocks on GPU (#35219)

Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
This commit is contained in:
Vadim Gimpelson
2026-03-10 03:32:20 -07:00
committed by GitHub
parent 507ddbe992
commit 4ff8c3c8f9
10 changed files with 287 additions and 8 deletions
+27
View File
@@ -197,6 +197,7 @@ from vllm.v1.worker.workspace import lock_workspace
from .utils import (
AttentionGroup,
KVBlockZeroer,
add_kv_sharing_layers_to_kv_cache_groups,
bind_kv_cache,
prepare_kernel_block_sizes,
@@ -982,6 +983,26 @@ class GPUModelRunner(
decode_threshold=self.reorder_batch_threshold,
)
def _init_kv_zero_meta(self) -> None:
"""One-time precomputation for _zero_block_ids.
Delegates to KVBlockZeroer.init_meta with the runner's state.
Called from gpu_worker.py outside the CuMem pool context.
"""
self._kv_block_zeroer = KVBlockZeroer(self.device, self.pin_memory)
self._kv_block_zeroer.init_meta(
attn_groups_iter=self._kv_cache_spec_attn_group_iterator(),
kernel_block_sizes=self._kernel_block_sizes,
cache_dtype=self.cache_config.cache_dtype,
runner_only_attn_layers=self.runner_only_attn_layers,
static_forward_context=(self.compilation_config.static_forward_context),
)
def _zero_block_ids(self, block_ids: list[int]) -> None:
"""Zero the KV cache memory for the given block IDs."""
if hasattr(self, "_kv_block_zeroer"):
self._kv_block_zeroer.zero_block_ids(block_ids)
# Note: used for model runner override.
def _init_device_properties(self) -> None:
"""Initialize attributes from torch.cuda.get_device_properties"""
@@ -1018,6 +1039,11 @@ class GPUModelRunner(
for req_id in scheduler_output.finished_req_ids:
self.input_batch.remove_request(req_id)
# Zero GPU memory for freshly allocated cache blocks to prevent
# stale NaN/data from corrupting attention or SSM computation.
if scheduler_output.new_block_ids_to_zero:
self._zero_block_ids(scheduler_output.new_block_ids_to_zero)
# Free the cached encoder outputs.
for mm_hash in scheduler_output.free_encoder_mm_hashes:
self.encoder_cache.pop(mm_hash, None)
@@ -6476,6 +6502,7 @@ class GPUModelRunner(
kernel_block_sizes = prepare_kernel_block_sizes(
kv_cache_config, self.attn_groups
)
self._kernel_block_sizes = kernel_block_sizes
# create metadata builders
self.initialize_metadata_builders(kv_cache_config, kernel_block_sizes)