[Bugfix][HiSparse] Size the KV cache from the groups HiSparse allocates (#59450)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Lucas Wilkinson
parent
1a9eaa3a68
commit
f7999d2e44
@@ -472,9 +472,11 @@ class HiSparseResidentSpec(KVCacheSpec):
|
||||
return cdiv(num_tokens, self.block_size)
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
return cdiv(vllm_config.model_config.max_model_len, self.block_size) * (
|
||||
self.page_size
|
||||
max_blocks = self.max_admission_blocks_per_request(
|
||||
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
|
||||
max_model_len=vllm_config.model_config.max_model_len,
|
||||
)
|
||||
return max_blocks * self.page_size
|
||||
|
||||
@property
|
||||
def has_layer_views(self) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user