[Misc] Remove deprecated calculate_kv_scales runtime KV scale calculation (#49389)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
This commit is contained in:
@@ -144,11 +144,6 @@ def set_default_quant_scales(layer: nn.Module, register_buffer: bool = False) ->
|
||||
layer._v_scale_cpu = torch.tensor(1.0, dtype=torch.float32)
|
||||
layer._prob_scale_float = 1.0
|
||||
|
||||
# Initialize q/k/v range constants used by calc_kv_scales
|
||||
layer.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32)
|
||||
layer.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
|
||||
layer.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
|
||||
|
||||
|
||||
def _init_kv_cache_quant(
|
||||
layer: nn.Module,
|
||||
@@ -270,10 +265,8 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
vllm_config = get_current_vllm_config()
|
||||
if cache_config is not None:
|
||||
kv_cache_dtype = cache_config.cache_dtype
|
||||
calculate_kv_scales = cache_config.calculate_kv_scales
|
||||
else:
|
||||
kv_cache_dtype = "auto"
|
||||
calculate_kv_scales = False
|
||||
|
||||
# llm-compressor models declare an FP8 KV-cache scheme in their
|
||||
# checkpoint config. Honor it only when the user did not explicitly
|
||||
@@ -284,10 +277,8 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
kv_cache_scheme = getattr(quant_config, "kv_cache_scheme", None)
|
||||
if kv_cache_scheme is not None and kv_cache_dtype == "auto":
|
||||
kv_cache_dtype = "fp8"
|
||||
calculate_kv_scales = False
|
||||
if cache_config is not None:
|
||||
cache_config.cache_dtype = "fp8"
|
||||
cache_config.calculate_kv_scales = False
|
||||
|
||||
# Check if per-head quant scales are required based on kv_cache_scheme
|
||||
use_per_head_quant_scales = (
|
||||
@@ -312,7 +303,6 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
skip = True
|
||||
if skip:
|
||||
kv_cache_dtype = "auto"
|
||||
calculate_kv_scales = False
|
||||
logger.debug(
|
||||
"Layer %s: kv_cache_dtype=%s, sliding_window=%s",
|
||||
prefix,
|
||||
@@ -324,7 +314,6 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
kv_cache_dtype, vllm_config.model_config
|
||||
)
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
self.calculate_kv_scales = calculate_kv_scales
|
||||
if num_kv_heads is None:
|
||||
num_kv_heads = num_heads
|
||||
assert num_heads % num_kv_heads == 0, (
|
||||
@@ -505,10 +494,6 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
context using
|
||||
`vllm.forward_context.get_forward_context().attn_metadata`.
|
||||
"""
|
||||
if self.calculate_kv_scales:
|
||||
torch.ops.vllm.maybe_calc_kv_scales(
|
||||
query, key, value, _encode_layer_name(self.layer_name)
|
||||
)
|
||||
if output_dtype is None:
|
||||
output_dtype = query.dtype
|
||||
if self.query_quant is not None:
|
||||
@@ -581,18 +566,6 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
return output.view(-1, hidden_size)
|
||||
|
||||
def calc_kv_scales(self, query, key, value):
|
||||
self._q_scale.copy_(torch.abs(query).max() / self.q_range)
|
||||
self._k_scale.copy_(torch.abs(key).max() / self.k_range)
|
||||
self._v_scale.copy_(torch.abs(value).max() / self.v_range)
|
||||
self._q_scale_float = self._q_scale.item()
|
||||
self._k_scale_float = self._k_scale.item()
|
||||
self._v_scale_float = self._v_scale.item()
|
||||
self._k_scale_cpu.fill_(self._k_scale_float)
|
||||
self._v_scale_cpu.fill_(self._v_scale_float)
|
||||
# We only calculate the scales once
|
||||
self.calculate_kv_scales = False
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
s = f"head_size={self.impl.head_size}" # type: ignore
|
||||
s += f", num_heads={self.impl.num_heads}" # type: ignore
|
||||
@@ -694,41 +667,6 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
|
||||
|
||||
def maybe_calc_kv_scales(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
|
||||
# Only calculate if the layer's calculate_kv_scales flag is True
|
||||
# This flag gets set to False after the first forward pass
|
||||
if not self.calculate_kv_scales:
|
||||
return
|
||||
|
||||
self.calc_kv_scales(query, key, value)
|
||||
|
||||
|
||||
def maybe_calc_kv_scales_fake(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="maybe_calc_kv_scales",
|
||||
op_func=maybe_calc_kv_scales,
|
||||
mutates_args=["query", "key", "value"],
|
||||
fake_impl=maybe_calc_kv_scales_fake,
|
||||
)
|
||||
|
||||
|
||||
def get_attention_context(
|
||||
layer_name: str,
|
||||
) -> tuple[Any, "Attention | MLAAttention", torch.Tensor, torch.Tensor]:
|
||||
|
||||
@@ -399,10 +399,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
if cache_config is not None:
|
||||
kv_cache_dtype: CacheDType = cache_config.cache_dtype
|
||||
calculate_kv_scales = cache_config.calculate_kv_scales
|
||||
else:
|
||||
kv_cache_dtype = "auto"
|
||||
calculate_kv_scales = False
|
||||
self.quant_config = quant_config
|
||||
|
||||
if cache_config is not None and cache_config.kv_cache_dtype_skip_layers:
|
||||
@@ -411,7 +409,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
if str(layer_idx) in cache_config.kv_cache_dtype_skip_layers:
|
||||
kv_cache_dtype = "auto"
|
||||
calculate_kv_scales = False
|
||||
logger.debug(
|
||||
"Layer %s: kv_cache_dtype=%s",
|
||||
prefix,
|
||||
@@ -460,7 +457,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
# Initialize KV cache quantization attributes
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
self.calculate_kv_scales = calculate_kv_scales
|
||||
_init_kv_cache_quant(self, quant_config, prefix)
|
||||
|
||||
if (
|
||||
@@ -560,11 +556,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
and _vllm_config.parallel_config.dcp_comm_backend == "a2a"
|
||||
)
|
||||
|
||||
# Initialize q/k/v range constants.
|
||||
self.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32)
|
||||
self.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
|
||||
self.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
|
||||
|
||||
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
|
||||
|
||||
# If kv_b_proj_weight is unquantized, quantize it to mxfp4 if supported
|
||||
@@ -606,14 +597,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
output_shape: torch.Size | None = None,
|
||||
q_dcp_replicated: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.calculate_kv_scales:
|
||||
torch.ops.vllm.maybe_calc_kv_scales(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
_encode_layer_name(self.layer_name),
|
||||
)
|
||||
|
||||
if self.use_direct_call:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
@@ -1110,30 +1093,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
if not should_load_quant_weights(quant_method):
|
||||
set_default_quant_scales(self, register_buffer=False)
|
||||
|
||||
def calc_kv_scales(
|
||||
self, q: torch.Tensor, kv_c_normed: torch.Tensor, k_pe: torch.Tensor
|
||||
) -> None:
|
||||
"""Optional scale calculation for MLA inputs.
|
||||
|
||||
Mirrors Attention.calc_kv_scales. Not all MLA backends require this
|
||||
"""
|
||||
# Use safe defaults if ranges are not present
|
||||
q_range = getattr(self, "q_range", torch.tensor(1.0))
|
||||
k_range = getattr(self, "k_range", torch.tensor(1.0))
|
||||
v_range = getattr(self, "v_range", torch.tensor(1.0))
|
||||
|
||||
self._q_scale.copy_(torch.abs(q).max() / q_range)
|
||||
# kv_c_normed is the compressed KV representation; use it for k/v
|
||||
kv_abs_max = torch.abs(kv_c_normed).max()
|
||||
self._k_scale.copy_(kv_abs_max / k_range)
|
||||
self._v_scale.copy_(kv_abs_max / v_range)
|
||||
self._q_scale_float = self._q_scale.item()
|
||||
self._k_scale_float = self._k_scale.item()
|
||||
self._v_scale_float = self._v_scale.item()
|
||||
self._k_scale_cpu.fill_(self._k_scale_float)
|
||||
self._v_scale_cpu.fill_(self._v_scale_float)
|
||||
self.calculate_kv_scales = False
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return self.attn_backend
|
||||
|
||||
|
||||
@@ -80,8 +80,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
return
|
||||
|
||||
# Per-token-head quantized KV cache: scales are computed dynamically
|
||||
# per (token, head) in the kernel at cache-write time. Checkpoint
|
||||
# scales are never used regardless of calculate_kv_scales.
|
||||
# per (token, head) in the kernel at cache-write time.
|
||||
if kv_cache_uses_per_token_head_scales(layer.kv_cache_dtype):
|
||||
layer._k_scale.copy_(1.0)
|
||||
layer._v_scale.copy_(1.0)
|
||||
@@ -97,10 +96,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
# regardless whether the kv-scale is available in the checkpoint.
|
||||
# No need to process kv scales after loading if we are going to
|
||||
# calculate them on the fly.
|
||||
if (
|
||||
is_quantized_kv_cache(layer.kv_cache_dtype)
|
||||
and not layer.calculate_kv_scales
|
||||
):
|
||||
if is_quantized_kv_cache(layer.kv_cache_dtype):
|
||||
if layer.k_scale > 0.0 and layer.v_scale > 0.0:
|
||||
# We prefer to use separate k_scale and v_scale if present
|
||||
k_scale = layer.k_scale.to("cpu").tolist()
|
||||
@@ -158,7 +154,6 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
q_scale = layer.q_scale
|
||||
if current_platform.is_fp8_fnuz():
|
||||
q_scale *= 2
|
||||
layer.calculate_kv_scales = False
|
||||
else:
|
||||
q_scale = 1.0
|
||||
if layer.prob_scale > 0.0:
|
||||
|
||||
@@ -445,23 +445,6 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
|
||||
Args:
|
||||
vllm_config: vLLM Config
|
||||
"""
|
||||
cache_config = vllm_config.cache_config
|
||||
|
||||
# Disable calculate_kv_scales for hybrid models: uninitialized
|
||||
# recurrent state corrupts scales during the calibration pass.
|
||||
# See issue: https://github.com/vllm-project/vllm/issues/37554
|
||||
|
||||
if cache_config.calculate_kv_scales:
|
||||
logger.warning(
|
||||
"Disabling calculate_kv_scales for hybrid model '%s'. "
|
||||
"Hybrid models with recurrent layers (GDN, Mamba, SSM) "
|
||||
"produce unreliable KV cache scales during the "
|
||||
"calibration pass because recurrent state is "
|
||||
"uninitialized. Using default scale of 1.0 instead.",
|
||||
vllm_config.model_config.model,
|
||||
)
|
||||
cache_config.calculate_kv_scales = False
|
||||
|
||||
# Enable FULL_AND_PIECEWISE by default
|
||||
MambaModelConfig.verify_and_update_config(vllm_config)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user