[Bugfix][Qwen4Exp] Release the profiling KV cache held by QSA key views (#58961)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Gabriel Wu
2026-09-28 08:11:46 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 66d2d4b2e7
commit a2be4d3cc7
2 changed files with 34 additions and 10 deletions
+12 -9
View File
@@ -825,15 +825,18 @@ class QSAKeyStateCache(_QSAStateCache):
)
super().__init__(head_size=storage_head_size, **kwargs)
def bind_kv_cache(self, kv_cache: torch.Tensor) -> None:
super().bind_kv_cache(kv_cache)
qsa_cache = self.kv_cache
self.key_cache = qsa_cache[..., : self.key_head_size]
if self.cache_rope_positions:
position_tail = qsa_cache[..., self.rope_position_offset :]
self.rope_position_cache = position_tail.view(torch.int64)
else:
self.rope_position_cache = None
# Derived on access so `kv_cache` stays the only reference to the bound
# storage: clearing it (e.g. after CUDA graph memory profiling) must free
# the cache, or the freed block stays pinned in the allocator.
@property
def key_cache(self) -> torch.Tensor:
return self.kv_cache[..., : self.key_head_size]
@property
def rope_position_cache(self) -> torch.Tensor | None:
if not self.cache_rope_positions:
return None
return self.kv_cache[..., self.rope_position_offset :].view(torch.int64)
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
# Hold the open group's committed keys plus every row a speculative