[Bugfix] Include inline per-token-head scales in offloaded page transfer width (#48411)

Signed-off-by: Itay Etelis <itay.etelis@ibm.com>
Signed-off-by: Itay Etelis <Itay.etelis@gmail.com>
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
Co-authored-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: Itay Etelis <Itay.etelis@gmail.com>
This commit is contained in:
Itay Etelis
2026-07-14 16:07:26 +03:00
committed by GitHub
co-authored by Itay Etelis Itay Etelis
parent 7a74a9662b
commit 32aef44388
3 changed files with 41 additions and 6 deletions
+9 -5
View File
@@ -182,19 +182,23 @@ class AttentionSpec(KVCacheSpec):
indexes_kv_by_block_stride: bool = False
@property
def page_size_bytes(self) -> int:
real_page_size = self.real_page_size_bytes
def unpadded_page_size_bytes(self) -> int:
unpadded = self.real_page_size_bytes
# Per-token-head scales are stored in separate tensors managed
# by the attention backend, but the memory is carved from the
# raw KV cache allocation so it must be budgeted here.
if self.kv_quant_mode.is_per_token_head:
real_page_size += (
unpadded += (
2 * self.block_size * self.num_kv_heads * get_dtype_size(torch.float32)
)
return unpadded
@property
def page_size_bytes(self) -> int:
if self.page_size_padded is not None:
assert self.page_size_padded >= real_page_size
assert self.page_size_padded >= self.unpadded_page_size_bytes
return self.page_size_padded
return real_page_size
return self.unpadded_page_size_bytes
@property
def real_page_size_bytes(self) -> int: