[Attn,KV-cache] Use per-head scales in the attention selector (#34281)

Signed-off-by: Your Name <you@example.com>
Signed-off-by: Eldar Kurtic <research@neuralmagic.com>
Co-authored-by: Eldar Kurtic <research@neuralmagic.com>
Co-authored-by: Your Name <you@example.com>
This commit is contained in:
Eldar Kurtić
2026-02-24 09:02:43 -05:00
committed by GitHub
co-authored by Eldar Kurtic Your Name
parent 761e63e541
commit a87cc50859
6 changed files with 80 additions and 15 deletions
@@ -985,14 +985,7 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
self.quant_config.kv_cache_scheme["strategy"]
)
if strategy == QuantizationStrategy.ATTN_HEAD:
assert layer.impl.supports_per_head_quant_scales, (
f"Layer {layer.__class__.__name__} with implementation "
f"{layer.impl.__class__.__name__} does not support per-head scales."
)
n_scales = int(layer.num_kv_heads)
else:
n_scales = 1
n_scales = int(layer.num_kv_heads) if strategy == "attn_head" else 1
layer.k_scale = torch.nn.Parameter(
torch.ones(n_scales, requires_grad=False, dtype=torch.float32)