[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-09 21:25:44 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9bd074a996
commit d5f356dae0
15 changed files with 429 additions and 128 deletions
+22 -6
View File
@@ -214,10 +214,26 @@ def bind_hisparse_kv_caches(
for tensor_config in kv_cache_config.kv_cache_tensors
for layer_index, name in enumerate(tensor_config.layers)
}
resident_source_index = 0
for group_id, group in enumerate(kv_cache_config.kv_cache_groups):
if not isinstance(group.kv_cache_spec, HiSparseResidentSpec):
continue
resident_groups = [
(group_id, group)
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
if isinstance(group.kv_cache_spec, HiSparseResidentSpec)
]
residency: torch.Tensor | None = None
if resident_groups:
table_shapes = {
block_tables.input_block_tables[group_id].shape
for group_id, _ in resident_groups
}
assert len(table_shapes) == 1
max_num_reqs, max_num_pages = table_shapes.pop()
residency = torch.zeros(
(max_num_reqs, len(resident_groups), max_num_pages),
dtype=torch.int32,
device=block_tables.input_block_tables[resident_groups[0][0]].device,
)
for resident_source_index, (group_id, group) in enumerate(resident_groups):
assert residency is not None
for cache_name in group.layer_names:
assert cache_name.endswith(HISPARSE_RESIDENT_SUFFIX)
layer_name = cache_name[: -len(HISPARSE_RESIDENT_SUFFIX)]
@@ -230,13 +246,13 @@ def bind_hisparse_kv_caches(
block_stride=tensor_config.block_stride,
num_blocks=kv_cache_config.num_blocks,
block_size=group.kv_cache_spec.block_size,
block_table=block_tables.input_block_tables[group_id],
block_table=residency[:, resident_source_index],
slot_mapping=block_tables.slot_mappings[group_id],
)
cache_handle.residency = residency
assert cache_handle.view is not None
kv_caches[cache_name] = cache_handle.view.cache
cache_handle.runtime.resident_source_index = resident_source_index
resident_source_index += 1
hot_backing: torch.Tensor | None = None
cache_handles: list[HiSparseCacheHandle] = []