[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9bd074a996
commit
d5f356dae0
@@ -214,10 +214,26 @@ def bind_hisparse_kv_caches(
|
||||
for tensor_config in kv_cache_config.kv_cache_tensors
|
||||
for layer_index, name in enumerate(tensor_config.layers)
|
||||
}
|
||||
resident_source_index = 0
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
if not isinstance(group.kv_cache_spec, HiSparseResidentSpec):
|
||||
continue
|
||||
resident_groups = [
|
||||
(group_id, group)
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
if isinstance(group.kv_cache_spec, HiSparseResidentSpec)
|
||||
]
|
||||
residency: torch.Tensor | None = None
|
||||
if resident_groups:
|
||||
table_shapes = {
|
||||
block_tables.input_block_tables[group_id].shape
|
||||
for group_id, _ in resident_groups
|
||||
}
|
||||
assert len(table_shapes) == 1
|
||||
max_num_reqs, max_num_pages = table_shapes.pop()
|
||||
residency = torch.zeros(
|
||||
(max_num_reqs, len(resident_groups), max_num_pages),
|
||||
dtype=torch.int32,
|
||||
device=block_tables.input_block_tables[resident_groups[0][0]].device,
|
||||
)
|
||||
for resident_source_index, (group_id, group) in enumerate(resident_groups):
|
||||
assert residency is not None
|
||||
for cache_name in group.layer_names:
|
||||
assert cache_name.endswith(HISPARSE_RESIDENT_SUFFIX)
|
||||
layer_name = cache_name[: -len(HISPARSE_RESIDENT_SUFFIX)]
|
||||
@@ -230,13 +246,13 @@ def bind_hisparse_kv_caches(
|
||||
block_stride=tensor_config.block_stride,
|
||||
num_blocks=kv_cache_config.num_blocks,
|
||||
block_size=group.kv_cache_spec.block_size,
|
||||
block_table=block_tables.input_block_tables[group_id],
|
||||
block_table=residency[:, resident_source_index],
|
||||
slot_mapping=block_tables.slot_mappings[group_id],
|
||||
)
|
||||
cache_handle.residency = residency
|
||||
assert cache_handle.view is not None
|
||||
kv_caches[cache_name] = cache_handle.view.cache
|
||||
cache_handle.runtime.resident_source_index = resident_source_index
|
||||
resident_source_index += 1
|
||||
|
||||
hot_backing: torch.Tensor | None = None
|
||||
cache_handles: list[HiSparseCacheHandle] = []
|
||||
|
||||
Reference in New Issue
Block a user