Remove KV cache scale boilerplate from model weight loading methods (#43167)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-06-05 05:19:04 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7fe7800fa4
commit 62215e72c6
56 changed files with 87 additions and 730 deletions
@@ -15,6 +15,30 @@ from vllm.v1.kv_cache_interface import kv_cache_uses_per_token_head_scales
logger = init_logger(__name__)
class KVCacheScaleParameter(torch.nn.Parameter):
"""Scalar parameter for KV-cache scales.
Initialized to -1.0 (an invalid sentinel) so call sites just write
`KVCacheScaleParameter()`. The `weight_loader` accepts shape `()` or
`(1,)` and rejects anything else — per-head scales go through a separate
path (compressed-tensors' `_tp_aware_loader`), not this one. Per-instance
overrides still work because instance attribute assignment shadows this
class-level loader.
"""
def __new__(cls) -> "KVCacheScaleParameter":
return super().__new__(cls, torch.tensor(-1.0), requires_grad=False)
@staticmethod
def weight_loader(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None:
if loaded_weight.numel() != 1:
raise ValueError(
f"KV-cache scale expects a scalar weight, got shape "
f"{tuple(loaded_weight.shape)}"
)
param.data.copy_(loaded_weight.reshape(()))
class BaseKVCacheMethod(QuantizeMethodBase):
"""
Quant method that adds `_k_scale` and `_v_scale` attributes to the
@@ -37,11 +61,11 @@ class BaseKVCacheMethod(QuantizeMethodBase):
# Initialize the Q and KV cache scales to -1.0, an invalid value.
# If the q and k/v_scales appear in the checkpoint, it will be
# overwritten when loading weights.
layer.q_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer.k_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer.v_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer.q_scale = KVCacheScaleParameter()
layer.k_scale = KVCacheScaleParameter()
layer.v_scale = KVCacheScaleParameter()
# Initialize P = softmax(QK^T) scales
layer.prob_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer.prob_scale = KVCacheScaleParameter()
def apply(self, layer: torch.nn.Module) -> torch.Tensor:
raise RuntimeError(f"{self.__class__.__name__}.apply should not be called.")