[Misc] Remove deprecated calculate_kv_scales runtime KV scale calculation (#49389)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
This commit is contained in:
wangxiyuan
2026-08-03 11:15:13 -04:00
committed by GitHub
parent 0cf49a5d15
commit dd11df04f3
19 changed files with 16 additions and 263 deletions
@@ -144,11 +144,6 @@ def set_default_quant_scales(layer: nn.Module, register_buffer: bool = False) ->
layer._v_scale_cpu = torch.tensor(1.0, dtype=torch.float32)
layer._prob_scale_float = 1.0
# Initialize q/k/v range constants used by calc_kv_scales
layer.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32)
layer.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
layer.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
def _init_kv_cache_quant(
layer: nn.Module,
@@ -270,10 +265,8 @@ class Attention(nn.Module, AttentionLayerBase):
vllm_config = get_current_vllm_config()
if cache_config is not None:
kv_cache_dtype = cache_config.cache_dtype
calculate_kv_scales = cache_config.calculate_kv_scales
else:
kv_cache_dtype = "auto"
calculate_kv_scales = False
# llm-compressor models declare an FP8 KV-cache scheme in their
# checkpoint config. Honor it only when the user did not explicitly
@@ -284,10 +277,8 @@ class Attention(nn.Module, AttentionLayerBase):
kv_cache_scheme = getattr(quant_config, "kv_cache_scheme", None)
if kv_cache_scheme is not None and kv_cache_dtype == "auto":
kv_cache_dtype = "fp8"
calculate_kv_scales = False
if cache_config is not None:
cache_config.cache_dtype = "fp8"
cache_config.calculate_kv_scales = False
# Check if per-head quant scales are required based on kv_cache_scheme
use_per_head_quant_scales = (
@@ -312,7 +303,6 @@ class Attention(nn.Module, AttentionLayerBase):
skip = True
if skip:
kv_cache_dtype = "auto"
calculate_kv_scales = False
logger.debug(
"Layer %s: kv_cache_dtype=%s, sliding_window=%s",
prefix,
@@ -324,7 +314,6 @@ class Attention(nn.Module, AttentionLayerBase):
kv_cache_dtype, vllm_config.model_config
)
self.kv_cache_dtype = kv_cache_dtype
self.calculate_kv_scales = calculate_kv_scales
if num_kv_heads is None:
num_kv_heads = num_heads
assert num_heads % num_kv_heads == 0, (
@@ -505,10 +494,6 @@ class Attention(nn.Module, AttentionLayerBase):
context using
`vllm.forward_context.get_forward_context().attn_metadata`.
"""
if self.calculate_kv_scales:
torch.ops.vllm.maybe_calc_kv_scales(
query, key, value, _encode_layer_name(self.layer_name)
)
if output_dtype is None:
output_dtype = query.dtype
if self.query_quant is not None:
@@ -581,18 +566,6 @@ class Attention(nn.Module, AttentionLayerBase):
)
return output.view(-1, hidden_size)
def calc_kv_scales(self, query, key, value):
self._q_scale.copy_(torch.abs(query).max() / self.q_range)
self._k_scale.copy_(torch.abs(key).max() / self.k_range)
self._v_scale.copy_(torch.abs(value).max() / self.v_range)
self._q_scale_float = self._q_scale.item()
self._k_scale_float = self._k_scale.item()
self._v_scale_float = self._v_scale.item()
self._k_scale_cpu.fill_(self._k_scale_float)
self._v_scale_cpu.fill_(self._v_scale_float)
# We only calculate the scales once
self.calculate_kv_scales = False
def extra_repr(self) -> str:
s = f"head_size={self.impl.head_size}" # type: ignore
s += f", num_heads={self.impl.num_heads}" # type: ignore
@@ -694,41 +667,6 @@ class Attention(nn.Module, AttentionLayerBase):
)
def maybe_calc_kv_scales(
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
layer_name: LayerNameType,
) -> None:
layer_name = _resolve_layer_name(layer_name)
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
# Only calculate if the layer's calculate_kv_scales flag is True
# This flag gets set to False after the first forward pass
if not self.calculate_kv_scales:
return
self.calc_kv_scales(query, key, value)
def maybe_calc_kv_scales_fake(
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
layer_name: LayerNameType,
) -> None:
return
direct_register_custom_op(
op_name="maybe_calc_kv_scales",
op_func=maybe_calc_kv_scales,
mutates_args=["query", "key", "value"],
fake_impl=maybe_calc_kv_scales_fake,
)
def get_attention_context(
layer_name: str,
) -> tuple[Any, "Attention | MLAAttention", torch.Tensor, torch.Tensor]:
@@ -399,10 +399,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
if cache_config is not None:
kv_cache_dtype: CacheDType = cache_config.cache_dtype
calculate_kv_scales = cache_config.calculate_kv_scales
else:
kv_cache_dtype = "auto"
calculate_kv_scales = False
self.quant_config = quant_config
if cache_config is not None and cache_config.kv_cache_dtype_skip_layers:
@@ -411,7 +409,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
layer_idx = extract_layer_index(prefix)
if str(layer_idx) in cache_config.kv_cache_dtype_skip_layers:
kv_cache_dtype = "auto"
calculate_kv_scales = False
logger.debug(
"Layer %s: kv_cache_dtype=%s",
prefix,
@@ -460,7 +457,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
# Initialize KV cache quantization attributes
self.kv_cache_dtype = kv_cache_dtype
self.calculate_kv_scales = calculate_kv_scales
_init_kv_cache_quant(self, quant_config, prefix)
if (
@@ -560,11 +556,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
and _vllm_config.parallel_config.dcp_comm_backend == "a2a"
)
# Initialize q/k/v range constants.
self.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32)
self.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
self.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
# If kv_b_proj_weight is unquantized, quantize it to mxfp4 if supported
@@ -606,14 +597,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
output_shape: torch.Size | None = None,
q_dcp_replicated: torch.Tensor | None = None,
) -> torch.Tensor:
if self.calculate_kv_scales:
torch.ops.vllm.maybe_calc_kv_scales(
q,
kv_c_normed,
k_pe,
_encode_layer_name(self.layer_name),
)
if self.use_direct_call:
forward_context: ForwardContext = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
@@ -1110,30 +1093,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
if not should_load_quant_weights(quant_method):
set_default_quant_scales(self, register_buffer=False)
def calc_kv_scales(
self, q: torch.Tensor, kv_c_normed: torch.Tensor, k_pe: torch.Tensor
) -> None:
"""Optional scale calculation for MLA inputs.
Mirrors Attention.calc_kv_scales. Not all MLA backends require this
"""
# Use safe defaults if ranges are not present
q_range = getattr(self, "q_range", torch.tensor(1.0))
k_range = getattr(self, "k_range", torch.tensor(1.0))
v_range = getattr(self, "v_range", torch.tensor(1.0))
self._q_scale.copy_(torch.abs(q).max() / q_range)
# kv_c_normed is the compressed KV representation; use it for k/v
kv_abs_max = torch.abs(kv_c_normed).max()
self._k_scale.copy_(kv_abs_max / k_range)
self._v_scale.copy_(kv_abs_max / v_range)
self._q_scale_float = self._q_scale.item()
self._k_scale_float = self._k_scale.item()
self._v_scale_float = self._v_scale.item()
self._k_scale_cpu.fill_(self._k_scale_float)
self._v_scale_cpu.fill_(self._v_scale_float)
self.calculate_kv_scales = False
def get_attn_backend(self) -> type[AttentionBackend]:
return self.attn_backend
@@ -80,8 +80,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
return
# Per-token-head quantized KV cache: scales are computed dynamically
# per (token, head) in the kernel at cache-write time. Checkpoint
# scales are never used regardless of calculate_kv_scales.
# per (token, head) in the kernel at cache-write time.
if kv_cache_uses_per_token_head_scales(layer.kv_cache_dtype):
layer._k_scale.copy_(1.0)
layer._v_scale.copy_(1.0)
@@ -97,10 +96,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
# regardless whether the kv-scale is available in the checkpoint.
# No need to process kv scales after loading if we are going to
# calculate them on the fly.
if (
is_quantized_kv_cache(layer.kv_cache_dtype)
and not layer.calculate_kv_scales
):
if is_quantized_kv_cache(layer.kv_cache_dtype):
if layer.k_scale > 0.0 and layer.v_scale > 0.0:
# We prefer to use separate k_scale and v_scale if present
k_scale = layer.k_scale.to("cpu").tolist()
@@ -158,7 +154,6 @@ class BaseKVCacheMethod(QuantizeMethodBase):
q_scale = layer.q_scale
if current_platform.is_fp8_fnuz():
q_scale *= 2
layer.calculate_kv_scales = False
else:
q_scale = 1.0
if layer.prob_scale > 0.0:
-17
View File
@@ -445,23 +445,6 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
Args:
vllm_config: vLLM Config
"""
cache_config = vllm_config.cache_config
# Disable calculate_kv_scales for hybrid models: uninitialized
# recurrent state corrupts scales during the calibration pass.
# See issue: https://github.com/vllm-project/vllm/issues/37554
if cache_config.calculate_kv_scales:
logger.warning(
"Disabling calculate_kv_scales for hybrid model '%s'. "
"Hybrid models with recurrent layers (GDN, Mamba, SSM) "
"produce unreliable KV cache scales during the "
"calibration pass because recurrent state is "
"uninitialized. Using default scale of 1.0 instead.",
vllm_config.model_config.model,
)
cache_config.calculate_kv_scales = False
# Enable FULL_AND_PIECEWISE by default
MambaModelConfig.verify_and_update_config(vllm_config)