Remove KV cache scale boilerplate from model weight loading methods (#43167)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-06-05 05:19:04 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7fe7800fa4
commit 62215e72c6
56 changed files with 87 additions and 730 deletions
@@ -162,7 +162,13 @@ class QuantizationConfig(ABC):
"""
raise NotImplementedError
def get_cache_scale(self, name: str) -> str | None:
def get_cache_scale_mapper(self) -> "WeightsMapper | None":
"""Mapping from checkpoint KV-cache scale names to vLLM scale names.
Returning a mapper here causes `AutoWeightsLoader` to apply it to the
weight stream automatically; individual model `load_weights` methods
do not need to know about KV-cache scales.
"""
return None
def apply_vllm_mapper( # noqa: B027
+11 -18
View File
@@ -207,25 +207,18 @@ class Fp8Config(QuantizationConfig):
return Fp8KVCacheMethod(self)
return None
def get_cache_scale(self, name: str) -> str | None:
"""
Check whether the param name matches the format for k/v cache scales
in compressed-tensors. If this is the case, return its equivalent
param name expected by vLLM
def get_cache_scale_mapper(self) -> "WeightsMapper":
"""Map compressed-tensors KV-cache scale names to vLLM names."""
from vllm.model_executor.models.utils import WeightsMapper
:param name: param name
:return: matching param name for KV cache scale in vLLM
"""
if name.endswith(".output_scale") and ".k_proj" in name:
return name.replace(".k_proj.output_scale", ".attn.k_scale")
if name.endswith(".output_scale") and ".v_proj" in name:
return name.replace(".v_proj.output_scale", ".attn.v_scale")
if name.endswith(".output_scale") and ".q_proj" in name:
return name.replace(".q_proj.output_scale", ".attn.q_scale")
if name.endswith("self_attn.prob_output_scale"):
return name.replace(".prob_output_scale", ".attn.prob_scale")
# If no matches, return None
return None
return WeightsMapper(
orig_to_new_suffix={
".k_proj.output_scale": ".attn.k_scale",
".v_proj.output_scale": ".attn.v_scale",
".q_proj.output_scale": ".attn.q_scale",
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
}
)
class CopyNumelCounter(TorchDispatchMode):
@@ -15,6 +15,30 @@ from vllm.v1.kv_cache_interface import kv_cache_uses_per_token_head_scales
logger = init_logger(__name__)
class KVCacheScaleParameter(torch.nn.Parameter):
"""Scalar parameter for KV-cache scales.
Initialized to -1.0 (an invalid sentinel) so call sites just write
`KVCacheScaleParameter()`. The `weight_loader` accepts shape `()` or
`(1,)` and rejects anything else — per-head scales go through a separate
path (compressed-tensors' `_tp_aware_loader`), not this one. Per-instance
overrides still work because instance attribute assignment shadows this
class-level loader.
"""
def __new__(cls) -> "KVCacheScaleParameter":
return super().__new__(cls, torch.tensor(-1.0), requires_grad=False)
@staticmethod
def weight_loader(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None:
if loaded_weight.numel() != 1:
raise ValueError(
f"KV-cache scale expects a scalar weight, got shape "
f"{tuple(loaded_weight.shape)}"
)
param.data.copy_(loaded_weight.reshape(()))
class BaseKVCacheMethod(QuantizeMethodBase):
"""
Quant method that adds `_k_scale` and `_v_scale` attributes to the
@@ -37,11 +61,11 @@ class BaseKVCacheMethod(QuantizeMethodBase):
# Initialize the Q and KV cache scales to -1.0, an invalid value.
# If the q and k/v_scales appear in the checkpoint, it will be
# overwritten when loading weights.
layer.q_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer.k_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer.v_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer.q_scale = KVCacheScaleParameter()
layer.k_scale = KVCacheScaleParameter()
layer.v_scale = KVCacheScaleParameter()
# Initialize P = softmax(QK^T) scales
layer.prob_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer.prob_scale = KVCacheScaleParameter()
def apply(self, layer: torch.nn.Module) -> torch.Tensor:
raise RuntimeError(f"{self.__class__.__name__}.apply should not be called.")
@@ -646,26 +646,16 @@ class QuarkConfig(QuantizationConfig):
return scheme
def get_cache_scale(self, name: str) -> str | None:
"""
Check whether the param name matches the format for k/v cache scales
in quark. If this is the case, return its equivalent param name
expected by vLLM
:param name: param name
:return: matching param name for KV cache scale in vLLM
"""
if name.endswith(".output_scale") and ".k_proj" in name:
return name.replace(".k_proj.output_scale", ".attn.k_scale")
if name.endswith(".output_scale") and ".v_proj" in name:
return name.replace(".v_proj.output_scale", ".attn.v_scale")
if name.endswith(".output_scale") and ".q_proj" in name:
return name.replace(".q_proj.output_scale", ".attn.q_scale")
if name.endswith("self_attn.prob_output_scale"):
return name.replace(".prob_output_scale", ".attn.prob_scale")
# If no matches, return None
return None
def get_cache_scale_mapper(self) -> "WeightsMapper":
"""Map Quark KV-cache scale names to vLLM names."""
return WeightsMapper(
orig_to_new_suffix={
".k_proj.output_scale": ".attn.k_scale",
".v_proj.output_scale": ".attn.v_scale",
".q_proj.output_scale": ".attn.q_scale",
".self_attn.prob_output_scale": ".self_attn.attn.prob_scale",
}
)
class QuarkLinearMethod(LinearMethodBase):