[Attn,KV-cache] Use per-head scales in the attention selector (#34281)
Signed-off-by: Your Name <you@example.com> Signed-off-by: Eldar Kurtic <research@neuralmagic.com> Co-authored-by: Eldar Kurtic <research@neuralmagic.com> Co-authored-by: Your Name <you@example.com>
This commit is contained in:
co-authored by
Eldar Kurtic
Your Name
parent
761e63e541
commit
a87cc50859
@@ -229,13 +229,20 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
calculate_kv_scales = False
|
||||
|
||||
# llm-compressor mdls need to set cache_dtype to "fp8" manually.
|
||||
if getattr(quant_config, "kv_cache_scheme", None) is not None:
|
||||
kv_cache_scheme = getattr(quant_config, "kv_cache_scheme", None)
|
||||
if kv_cache_scheme is not None:
|
||||
kv_cache_dtype = "fp8"
|
||||
calculate_kv_scales = False
|
||||
if cache_config is not None:
|
||||
cache_config.cache_dtype = "fp8"
|
||||
cache_config.calculate_kv_scales = False
|
||||
|
||||
# Check if per-head quant scales are required based on kv_cache_scheme
|
||||
use_per_head_quant_scales = (
|
||||
kv_cache_scheme is not None
|
||||
and kv_cache_scheme.get("strategy") == "attn_head"
|
||||
)
|
||||
|
||||
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
|
||||
kv_cache_dtype, vllm_config.model_config
|
||||
)
|
||||
@@ -272,6 +279,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
use_mla=False,
|
||||
has_sink=self.has_sink,
|
||||
use_mm_prefix=self.use_mm_prefix,
|
||||
use_per_head_quant_scales=use_per_head_quant_scales,
|
||||
attn_type=attn_type,
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -985,14 +985,7 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
|
||||
self.quant_config.kv_cache_scheme["strategy"]
|
||||
)
|
||||
|
||||
if strategy == QuantizationStrategy.ATTN_HEAD:
|
||||
assert layer.impl.supports_per_head_quant_scales, (
|
||||
f"Layer {layer.__class__.__name__} with implementation "
|
||||
f"{layer.impl.__class__.__name__} does not support per-head scales."
|
||||
)
|
||||
n_scales = int(layer.num_kv_heads)
|
||||
else:
|
||||
n_scales = 1
|
||||
n_scales = int(layer.num_kv_heads) if strategy == "attn_head" else 1
|
||||
|
||||
layer.k_scale = torch.nn.Parameter(
|
||||
torch.ones(n_scales, requires_grad=False, dtype=torch.float32)
|
||||
|
||||
Reference in New Issue
Block a user