[Bugfix][KV Offload] Skip non-prefix-cacheable groups in SimpleCPUOffload (GLM-5.3-Flash kpool tail and QSA) (#56810)

Signed-off-by: Jared Wen <jaredwen@inferact.ai>
Signed-off-by: Jared Wen <w13431838023@gmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
This commit is contained in:
Jared Wen
2026-09-20 10:58:55 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 Zhewen Li
parent 1b9fa3eaa8
commit f648eed23d
3 changed files with 267 additions and 2 deletions
+9 -1
View File
@@ -165,6 +165,9 @@ class SimpleCPUOffloadScheduler:
kv_cache_config, offload_capacity
)
self.num_cpu_blocks = self.cpu_kv_cache_config.num_blocks
self.prefix_cacheable_group_ids = (
self.cpu_kv_cache_config.prefix_cacheable_group_ids
)
self.kv_event_medium = MEDIUM_STORAGE if disk_capacity_bytes > 0 else MEDIUM_CPU
# Find the full attention kv group for prefix cache matching.
self.fa_gidx = -1
@@ -297,7 +300,7 @@ class SimpleCPUOffloadScheduler:
"""GPU blocks to keep available (free/offloaded) per step in lazy mode."""
WATERMARK_RATIO = 1.0 # Reserve larger space to avoid running out of GPU blocks
target = 0
for g in kv_cache_config.kv_cache_groups:
for g in kv_cache_config.prefix_cacheable_groups:
spec = g.kv_cache_spec
# Only full attention is sharded across DCP ranks; replicated specs
# (mamba, sliding window, chunked-local) keep their own block size.
@@ -442,6 +445,9 @@ class SimpleCPUOffloadScheduler:
# the rest will be released along with the temp pin below.
cpu_hit_blocks: list[list[KVCacheBlock]] = []
for g in range(num_groups):
if g not in self.prefix_cacheable_group_ids:
cpu_hit_blocks.append([])
continue
g_block_size = self.group_block_sizes[g]
n_take_g = cdiv(num_external_tokens, g_block_size)
cpu_hit_blocks.append(cpu_hit_blocks_full[g][:n_take_g])
@@ -828,6 +834,8 @@ class SimpleCPUOffloadScheduler:
num_free = self.cpu_block_pool.get_num_free_blocks()
for g, group_gpu_ids in enumerate(block_ids_by_group):
if g not in self.prefix_cacheable_group_ids:
continue
if len(gpu_block_ids) >= num_free:
break
group_manager = self.cpu_coordinator.single_type_managers[g]