[Bugfix][HiSparse] Preserve per-layer offsets in KV cache bindings (#57027)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-09-15 20:27:11 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 18f8aa0465
commit 38ca7a899c
4 changed files with 48 additions and 6 deletions
+9 -6
View File
@@ -205,9 +205,12 @@ def bind_hisparse_kv_caches(
"""Bind existing cache storage and block tables; return the bound handles."""
assert host_pool.registered is not None
tensor_configs = {
name: tensor_config
name: (
tensor_config,
tensor_config.offset + layer_index * tensor_config.layer_stride,
)
for tensor_config in kv_cache_config.kv_cache_tensors
for name in tensor_config.layers
for layer_index, name in enumerate(tensor_config.layers)
}
resident_source_index = 0
for group_id, group in enumerate(kv_cache_config.kv_cache_groups):
@@ -216,12 +219,12 @@ def bind_hisparse_kv_caches(
for cache_name in group.layer_names:
assert cache_name.endswith(HISPARSE_RESIDENT_SUFFIX)
layer_name = cache_name[: -len(HISPARSE_RESIDENT_SUFFIX)]
tensor_config = tensor_configs[cache_name]
tensor_config, byte_offset = tensor_configs[cache_name]
assert not tensor_config.host_resident
cache_handle = _get_hisparse_cache(forward_context, layer_name)
cache_handle.bind_cache(
kv_caches[cache_name],
byte_offset=tensor_config.offset,
byte_offset=byte_offset,
block_stride=tensor_config.block_stride,
num_blocks=kv_cache_config.num_blocks,
block_size=group.kv_cache_spec.block_size,
@@ -249,11 +252,11 @@ def bind_hisparse_kv_caches(
raise RuntimeError("HiSparse hot tensors must share one GPU backing.")
layer_name = cache_name[: -len(HISPARSE_HOT_SUFFIX)]
cache_handle = _get_hisparse_cache(forward_context, layer_name)
tensor_config = tensor_configs[cache_name]
tensor_config, byte_offset = tensor_configs[cache_name]
assert not tensor_config.host_resident
cache_handle.runtime.bind_hot_cache(
raw_tensor,
byte_offset=tensor_config.offset,
byte_offset=byte_offset,
block_stride=tensor_config.block_stride,
num_blocks=kv_cache_config.num_blocks,
block_size=group.kv_cache_spec.block_size,