Remove KV cache scale boilerplate from model weight loading methods (#43167)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7fe7800fa4
commit
62215e72c6
@@ -15,6 +15,30 @@ from vllm.v1.kv_cache_interface import kv_cache_uses_per_token_head_scales
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class KVCacheScaleParameter(torch.nn.Parameter):
|
||||
"""Scalar parameter for KV-cache scales.
|
||||
|
||||
Initialized to -1.0 (an invalid sentinel) so call sites just write
|
||||
`KVCacheScaleParameter()`. The `weight_loader` accepts shape `()` or
|
||||
`(1,)` and rejects anything else — per-head scales go through a separate
|
||||
path (compressed-tensors' `_tp_aware_loader`), not this one. Per-instance
|
||||
overrides still work because instance attribute assignment shadows this
|
||||
class-level loader.
|
||||
"""
|
||||
|
||||
def __new__(cls) -> "KVCacheScaleParameter":
|
||||
return super().__new__(cls, torch.tensor(-1.0), requires_grad=False)
|
||||
|
||||
@staticmethod
|
||||
def weight_loader(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None:
|
||||
if loaded_weight.numel() != 1:
|
||||
raise ValueError(
|
||||
f"KV-cache scale expects a scalar weight, got shape "
|
||||
f"{tuple(loaded_weight.shape)}"
|
||||
)
|
||||
param.data.copy_(loaded_weight.reshape(()))
|
||||
|
||||
|
||||
class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
"""
|
||||
Quant method that adds `_k_scale` and `_v_scale` attributes to the
|
||||
@@ -37,11 +61,11 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
# Initialize the Q and KV cache scales to -1.0, an invalid value.
|
||||
# If the q and k/v_scales appear in the checkpoint, it will be
|
||||
# overwritten when loading weights.
|
||||
layer.q_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
|
||||
layer.k_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
|
||||
layer.v_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
|
||||
layer.q_scale = KVCacheScaleParameter()
|
||||
layer.k_scale = KVCacheScaleParameter()
|
||||
layer.v_scale = KVCacheScaleParameter()
|
||||
# Initialize P = softmax(QK^T) scales
|
||||
layer.prob_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
|
||||
layer.prob_scale = KVCacheScaleParameter()
|
||||
|
||||
def apply(self, layer: torch.nn.Module) -> torch.Tensor:
|
||||
raise RuntimeError(f"{self.__class__.__name__}.apply should not be called.")
|
||||
|
||||
Reference in New Issue
Block a user