[4/N] Expose HiSparse cache metrics via KV connector stats (#56061)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Nicolò Lucchesi
2026-09-12 10:47:55 -07:00
committed by GitHub
co-authored by Lucas Wilkinson OpenAI Codex
parent dff76bc3e8
commit 29332cf936
9 changed files with 308 additions and 8 deletions
+8
View File
@@ -464,6 +464,11 @@ class HiSparseIndexGroup:
)
self.logical_topk_ready = logical_topk_ready
self.followers: list[HiSparseRuntime] = []
self.swap_stats = torch.zeros(2, dtype=torch.uint64, device=device)
self.swap_stats_host = torch.empty(
2, dtype=torch.uint64, device="cpu", pin_memory=True
)
self.stats_row_bytes = 0
class HiSparseRuntime:
@@ -526,6 +531,7 @@ class HiSparseRuntime:
else:
index_group.followers.append(self)
self.index_group = index_group
index_group.stats_row_bytes += row_bytes
self.eager_host_mirror = config.eager_host_mirror
self.resident_source_index = -1
@@ -639,6 +645,7 @@ class HiSparseRuntime:
compute_stream.wait_stream(group.copy_stream)
group.device_global_indices.fill_(-1)
group.lru_slots.copy_(group.lru_init.expand_as(group.lru_slots))
group.swap_stats.zero_()
group.copy_stream.wait_stream(compute_stream)
def invalidate_sorted_slots(
@@ -741,6 +748,7 @@ class HiSparseRuntime:
request_state_indices,
self.region_stride,
None,
group.swap_stats,
physical_topk_indices,
hot.attention_block_stride,
req_id_per_token[:num_tokens].contiguous(),