[Bugfix][KV Offload] Skip non-prefix-cacheable groups in SimpleCPUOffload (GLM-5.3-Flash kpool tail and QSA) (#56810)
Signed-off-by: Jared Wen <jaredwen@inferact.ai> Signed-off-by: Jared Wen <w13431838023@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
This commit is contained in:
co-authored by
Claude Fable 5.1
Zhewen Li
parent
1b9fa3eaa8
commit
f648eed23d
@@ -165,6 +165,9 @@ class SimpleCPUOffloadScheduler:
|
||||
kv_cache_config, offload_capacity
|
||||
)
|
||||
self.num_cpu_blocks = self.cpu_kv_cache_config.num_blocks
|
||||
self.prefix_cacheable_group_ids = (
|
||||
self.cpu_kv_cache_config.prefix_cacheable_group_ids
|
||||
)
|
||||
self.kv_event_medium = MEDIUM_STORAGE if disk_capacity_bytes > 0 else MEDIUM_CPU
|
||||
# Find the full attention kv group for prefix cache matching.
|
||||
self.fa_gidx = -1
|
||||
@@ -297,7 +300,7 @@ class SimpleCPUOffloadScheduler:
|
||||
"""GPU blocks to keep available (free/offloaded) per step in lazy mode."""
|
||||
WATERMARK_RATIO = 1.0 # Reserve larger space to avoid running out of GPU blocks
|
||||
target = 0
|
||||
for g in kv_cache_config.kv_cache_groups:
|
||||
for g in kv_cache_config.prefix_cacheable_groups:
|
||||
spec = g.kv_cache_spec
|
||||
# Only full attention is sharded across DCP ranks; replicated specs
|
||||
# (mamba, sliding window, chunked-local) keep their own block size.
|
||||
@@ -442,6 +445,9 @@ class SimpleCPUOffloadScheduler:
|
||||
# the rest will be released along with the temp pin below.
|
||||
cpu_hit_blocks: list[list[KVCacheBlock]] = []
|
||||
for g in range(num_groups):
|
||||
if g not in self.prefix_cacheable_group_ids:
|
||||
cpu_hit_blocks.append([])
|
||||
continue
|
||||
g_block_size = self.group_block_sizes[g]
|
||||
n_take_g = cdiv(num_external_tokens, g_block_size)
|
||||
cpu_hit_blocks.append(cpu_hit_blocks_full[g][:n_take_g])
|
||||
@@ -828,6 +834,8 @@ class SimpleCPUOffloadScheduler:
|
||||
num_free = self.cpu_block_pool.get_num_free_blocks()
|
||||
|
||||
for g, group_gpu_ids in enumerate(block_ids_by_group):
|
||||
if g not in self.prefix_cacheable_group_ids:
|
||||
continue
|
||||
if len(gpu_block_ids) >= num_free:
|
||||
break
|
||||
group_manager = self.cpu_coordinator.single_type_managers[g]
|
||||
|
||||
Reference in New Issue
Block a user