From dd11df04f3b7046c40f13e586ac38a3725bc3c03 Mon Sep 17 00:00:00 2001 From: wangxiyuan Date: Mon, 3 Aug 2026 23:15:13 +0800 Subject: [PATCH] [Misc] Remove deprecated calculate_kv_scales runtime KV scale calculation (#49389) Signed-off-by: wangxiyuan --- .buildkite/test-amd.yaml | 2 +- .buildkite/test_areas/compile.yaml | 3 - .buildkite/test_areas/pytorch.yaml | 3 +- docs/design/metrics.md | 2 +- .../quantization/quantized_kvcache.md | 36 +---------- tests/compile/fullgraph/test_full_graph.py | 32 ---------- .../quantization/test_per_token_kv_cache.py | 4 -- tests/quantization/test_fp8.py | 28 +++------ tests/quantization/test_per_token_kv_cache.py | 1 - vllm/config/cache.py | 17 ----- vllm/engine/arg_utils.py | 5 -- vllm/envs.py | 6 -- .../layers/attention/attention.py | 62 ------------------- .../layers/attention/mla_attention.py | 41 ------------ .../layers/quantization/kv_cache.py | 9 +-- vllm/model_executor/models/config.py | 17 ----- vllm/models/minimax_m3/amd/model.py | 1 - vllm/models/minimax_m3/nvidia/model.py | 1 - vllm/v1/worker/gpu_model_runner.py | 9 --- 19 files changed, 16 insertions(+), 263 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 5845844dd3..de1a3ca667 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -636,7 +636,7 @@ steps: - tests/compile - vllm/platforms/rocm.py commands: - - pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile' + - pytest -v -s compile/fullgraph/test_full_graph.py - label: Pytorch Nightly Dependency Override Check # TBD timeout_in_minutes: 180 diff --git a/.buildkite/test_areas/compile.yaml b/.buildkite/test_areas/compile.yaml index eba37f657d..3a0ea308a9 100644 --- a/.buildkite/test_areas/compile.yaml +++ b/.buildkite/test_areas/compile.yaml @@ -90,9 +90,6 @@ steps: - pytest -v -s tests/compile/passes/test_silu_mul_quant_fusion.py # this runner has 2 GPUs available even though num_devices=2 is not set - pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py - # test_fp8_kv_scale_compile requires FlashAttention (not supported on default L4/L40) - # TODO(luka) move to H100 once pass tests run on H100 - - pytest -v -s tests/compile/fullgraph/test_full_graph.py::test_fp8_kv_scale_compile - label: Fusion E2E Quick (H100) key: fusion-e2e-quick-h100 diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index f53f399946..bf8014f6dd 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -211,8 +211,7 @@ steps: - vllm/v1/ - tests/compile commands: - # fp8 kv scales not supported on sm89, tested on Blackwell instead - - pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile' + - pytest -v -s compile/fullgraph/test_full_graph.py - label: Pytorch Nightly Dependency Override Check # 2min key: pytorch-nightly-dependency-override-check diff --git a/docs/design/metrics.md b/docs/design/metrics.md index 7b463b8750..c7bc85756e 100644 --- a/docs/design/metrics.md +++ b/docs/design/metrics.md @@ -367,7 +367,7 @@ We use this concept for the `vllm:cache_config_info` metric: ```text # HELP vllm:cache_config_info Information of the LLMEngine CacheConfig # TYPE vllm:cache_config_info gauge -vllm:cache_config_info{block_size="16",cache_dtype="auto",calculate_kv_scales="False",cpu_offload_gb="0",enable_prefix_caching="False",gpu_memory_utilization="0.9",...} 1.0 +vllm:cache_config_info{block_size="16",cache_dtype="auto",cpu_offload_gb="0",enable_prefix_caching="False",gpu_memory_utilization="0.9",...} 1.0 ``` However, `prometheus_client` has diff --git a/docs/features/quantization/quantized_kvcache.md b/docs/features/quantization/quantized_kvcache.md index 50b1c5c2df..d2a69e5c62 100644 --- a/docs/features/quantization/quantized_kvcache.md +++ b/docs/features/quantization/quantized_kvcache.md @@ -27,18 +27,9 @@ You can configure how the quantization scales are computed in vLLM using three d _Configure with:_ ```python kv_cache_dtype="fp8" - calculate_kv_scales=False ``` -2. **Random token calibration (on-the-fly):** - Scales are automatically estimated from a single batch of random tokens during warmup and then fixed. - _Configure with:_ - ```python - kv_cache_dtype="fp8" - calculate_kv_scales=True - ``` - -3. **[Recommended] Calibration with a dataset (via llm-compressor):** +2. **[Recommended] Calibration with a dataset (via llm-compressor):** Scales are estimated using a curated calibration dataset for maximum accuracy. This requires the [llm-compressor](https://github.com/vllm-project/llm-compressor) library. _See example below!_ @@ -79,7 +70,7 @@ llm = LLM( ## Examples -### 1. No Calibration (`kv_cache_dtype="fp8"`, `calculate_kv_scales=False`) +### 1. No Calibration (`kv_cache_dtype="fp8"`) All quantization scales are set to 1.0. @@ -90,7 +81,6 @@ sampling_params = SamplingParams(temperature=0.7, top_p=0.8) llm = LLM( model="meta-llama/Llama-2-7b-chat-hf", kv_cache_dtype="fp8", - calculate_kv_scales=False, ) prompt = "London is the capital of" out = llm.generate(prompt, sampling_params)[0].outputs[0].text @@ -99,27 +89,7 @@ print(out) --- -### 2. Random Token Calibration (`kv_cache_dtype="fp8"`, `calculate_kv_scales=True`) - -Scales are automatically estimated from a single batch of tokens during warmup. - -```python -from vllm import LLM, SamplingParams - -sampling_params = SamplingParams(temperature=0.7, top_p=0.8) -llm = LLM( - model="meta-llama/Llama-2-7b-chat-hf", - kv_cache_dtype="fp8", - calculate_kv_scales=True, -) -prompt = "London is the capital of" -out = llm.generate(prompt, sampling_params)[0].outputs[0].text -print(out) -``` - ---- - -### 3. **[Recommended] Calibration Using a Dataset (with `llm-compressor`)** +### 2. **[Recommended] Calibration Using a Dataset (with `llm-compressor`)** For the highest-quality quantization, we recommend calibrating against a dataset using `llm-compressor`. This enables advanced strategies such as per-attention-head quantization. diff --git a/tests/compile/fullgraph/test_full_graph.py b/tests/compile/fullgraph/test_full_graph.py index 0f3368c0db..3865fc4d03 100644 --- a/tests/compile/fullgraph/test_full_graph.py +++ b/tests/compile/fullgraph/test_full_graph.py @@ -13,7 +13,6 @@ from vllm import LLM, SamplingParams from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode, PassConfig from vllm.platforms import current_platform from vllm.utils.torch_utils import is_torch_equal_or_newer -from vllm.v1.attention.backends.registry import AttentionBackendEnum from ...utils import create_new_process_for_each_test @@ -182,37 +181,6 @@ def test_custom_compile_config( run_model(compilation_config, model, **model_kwargs) -@pytest.mark.parametrize( - "compilation_mode", - [CompilationMode.NONE, CompilationMode.VLLM_COMPILE], -) -@pytest.mark.parametrize( - "model, backend", - [ - ("Qwen/Qwen2-0.5B", None), # Standard attention model - ( - "deepseek-ai/DeepSeek-V2-Lite", - AttentionBackendEnum.FLASHINFER_MLA, - ), # MLA (Multi-head Latent Attention) model - ], -) -def test_fp8_kv_scale_compile( - compilation_mode: CompilationMode, - model: str, - backend: AttentionBackendEnum | None, -): - model_kwargs = { - "quantization": "fp8", - "kv_cache_dtype": "fp8_e4m3", - "calculate_kv_scales": True, - "max_model_len": 512, - } - if backend: - model_kwargs["attention_config"] = {"backend": backend.name} - - run_model(compilation_mode, model, **model_kwargs) - - def run_model( compile_config: CompilationMode | CompilationConfig, model: str, **model_kwargs ): diff --git a/tests/models/quantization/test_per_token_kv_cache.py b/tests/models/quantization/test_per_token_kv_cache.py index e18f38ffe8..2a13179d2a 100644 --- a/tests/models/quantization/test_per_token_kv_cache.py +++ b/tests/models/quantization/test_per_token_kv_cache.py @@ -52,9 +52,6 @@ def test_per_token_head_kv_cache_accuracy( ) -> None: """Compare logprobs between bf16 baseline and per-token-head quantized KV cache. - - Uses calculate_kv_scales (dynamic scale computation) since there are - no per-token-head calibrated checkpoints available yet. """ with monkeypatch.context() as m: m.setenv("TOKENIZERS_PARALLELISM", "true") @@ -80,7 +77,6 @@ def test_per_token_head_kv_cache_accuracy( tensor_parallel_size=tensor_parallel_size, enforce_eager=enforce_eager, kv_cache_dtype=kv_cache_dtype, - calculate_kv_scales=True, attention_config={"backend": backend}, ) as vllm_model: test_outputs = vllm_model.generate_greedy_logprobs( diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 55ad9c6ddb..a1c74259f8 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -18,7 +18,6 @@ from vllm.model_executor.kernels.linear.scaled_mm import ( MarlinFP8ScaledMMLinearKernel, ) from vllm.model_executor.layers.attention.attention import ( - Attention, set_default_quant_scales, ) from vllm.model_executor.layers.fused_moe import FusedMoEFactory @@ -433,8 +432,7 @@ def test_fp8_reloading( method.process_weights_after_loading(layer) -@pytest.mark.parametrize("source", ["checkpoint", "runtime_calc"]) -def test_kv_cache_scale_sync_to_host_copies(source): +def test_kv_cache_scale_sync_to_host_copies(): """Test device-to-host sync of the k/v quantization scales, for both the checkpoint-load and runtime-calc paths that produce them. """ @@ -442,23 +440,13 @@ def test_kv_cache_scale_sync_to_host_copies(source): set_default_quant_scales(layer, register_buffer=True) layer.kv_cache_dtype = "fp8" - if source == "checkpoint": - # Scales come from the checkpoint, so runtime calc is disabled. - layer.calculate_kv_scales = False - method = BaseKVCacheMethod(quant_config=None) - method.create_weights(layer) - # 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale. - checkpoint_scale = torch.tensor(0.3, dtype=torch.float32) - layer.k_scale.weight_loader(layer.k_scale, checkpoint_scale) - layer.v_scale.weight_loader(layer.v_scale, checkpoint_scale) - method.process_weights_after_loading(layer) - else: - # First forward computes distinct, non-unity scales from live k/v. - layer.calculate_kv_scales = True - query = torch.full((4, 8), 10.0) - key = torch.full((4, 8), 60.0) - value = torch.full((4, 8), 50.0) - Attention.calc_kv_scales(layer, query, key, value) + method = BaseKVCacheMethod(quant_config=None) + method.create_weights(layer) + # 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale. + checkpoint_scale = torch.tensor(0.3, dtype=torch.float32) + layer.k_scale.weight_loader(layer.k_scale, checkpoint_scale) + layer.v_scale.weight_loader(layer.v_scale, checkpoint_scale) + method.process_weights_after_loading(layer) assert layer._k_scale_float != 1.0 assert layer._v_scale_float != 1.0 diff --git a/tests/quantization/test_per_token_kv_cache.py b/tests/quantization/test_per_token_kv_cache.py index 9fa4d97add..4d61932118 100644 --- a/tests/quantization/test_per_token_kv_cache.py +++ b/tests/quantization/test_per_token_kv_cache.py @@ -497,7 +497,6 @@ def test_process_weights_sets_placeholder_scales(kv_cache_dtype: str): layer = MagicMock() layer.kv_cache_dtype = kv_cache_dtype - layer.calculate_kv_scales = False layer.k_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False) layer.v_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False) layer.q_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False) diff --git a/vllm/config/cache.py b/vllm/config/cache.py index d267e9af35..a383524916 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -108,11 +108,6 @@ class CacheConfig: security risk tolerance against the performance benefits before turning this on. - "xxhash_cbor" combines canonical CBOR serialization with xxHash for reproducible hashing. Requires the optional ``xxhash`` package.""" - calculate_kv_scales: bool = False - """Deprecated: This option is deprecated and will be removed in v0.19. - It enables dynamic calculation of `k_scale` and `v_scale` when - kv_cache_dtype is fp8. If `False`, the scales will be loaded from the model - checkpoint if available. Otherwise, the scales will default to 1.0.""" kv_cache_dtype_skip_layers: list[str] = field(default_factory=list) """Layer patterns to skip KV cache quantization. Accepts layer indices (e.g., '0', '2', '4') or attention type names (e.g., 'sliding_window').""" @@ -270,18 +265,6 @@ class CacheConfig: self.user_specified_mamba_block_size = True return self - @field_validator("calculate_kv_scales", mode="after") - @classmethod - def _warn_deprecated_calculate_kv_scales(cls, calculate_kv_scales: bool) -> bool: - if calculate_kv_scales: - logger.warning( - "The `--calculate-kv-scales` option is deprecated and will " - "be removed in v0.19. The scales will be loaded from the " - "model checkpoint if available, otherwise they default to " - "1.0." - ) - return calculate_kv_scales - @field_validator("cache_dtype", mode="after") @classmethod def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType: diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 3003d73859..7fb925aa60 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -695,7 +695,6 @@ class EngineArgs: override_attention_dtype: str | None = ModelConfig.override_attention_dtype attention_backend: AttentionBackendEnum | None = AttentionConfig.backend - calculate_kv_scales: bool = CacheConfig.calculate_kv_scales kv_cache_dtype_skip_layers: list[str] = get_field( CacheConfig, "kv_cache_dtype_skip_layers" ) @@ -1210,9 +1209,6 @@ class EngineArgs: cache_group.add_argument( "--prefix-caching-hash-algo", **cache_kwargs["prefix_caching_hash_algo"] ) - cache_group.add_argument( - "--calculate-kv-scales", **cache_kwargs["calculate_kv_scales"] - ) cache_group.add_argument( "--kv-cache-dtype-skip-layers", **cache_kwargs["kv_cache_dtype_skip_layers"] ) @@ -1957,7 +1953,6 @@ class EngineArgs: sliding_window=sliding_window, enable_prefix_caching=self.enable_prefix_caching, prefix_caching_hash_algo=self.prefix_caching_hash_algo, - calculate_kv_scales=self.calculate_kv_scales, kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers, kv_sharing_fast_prefill=self.kv_sharing_fast_prefill, mamba_cache_dtype=self.mamba_cache_dtype, diff --git a/vllm/envs.py b/vllm/envs.py index c77184161b..4c247e6612 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -1325,12 +1325,6 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB": lambda: maybe_convert_int( os.environ.get("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB", None) ), - # Divisor for dynamic query scale factor calculation for FP8 KV Cache - "Q_SCALE_CONSTANT": lambda: int(os.getenv("Q_SCALE_CONSTANT", "200")), - # Divisor for dynamic key scale factor calculation for FP8 KV Cache - "K_SCALE_CONSTANT": lambda: int(os.getenv("K_SCALE_CONSTANT", "200")), - # Divisor for dynamic value scale factor calculation for FP8 KV Cache - "V_SCALE_CONSTANT": lambda: int(os.getenv("V_SCALE_CONSTANT", "100")), # If set, enable multiprocessing in LLM for the V1 code path. "VLLM_ENABLE_V1_MULTIPROCESSING": lambda: bool( int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1")) diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index c8e5a9c6ac..4db42e0ecf 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -144,11 +144,6 @@ def set_default_quant_scales(layer: nn.Module, register_buffer: bool = False) -> layer._v_scale_cpu = torch.tensor(1.0, dtype=torch.float32) layer._prob_scale_float = 1.0 - # Initialize q/k/v range constants used by calc_kv_scales - layer.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32) - layer.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32) - layer.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32) - def _init_kv_cache_quant( layer: nn.Module, @@ -270,10 +265,8 @@ class Attention(nn.Module, AttentionLayerBase): vllm_config = get_current_vllm_config() if cache_config is not None: kv_cache_dtype = cache_config.cache_dtype - calculate_kv_scales = cache_config.calculate_kv_scales else: kv_cache_dtype = "auto" - calculate_kv_scales = False # llm-compressor models declare an FP8 KV-cache scheme in their # checkpoint config. Honor it only when the user did not explicitly @@ -284,10 +277,8 @@ class Attention(nn.Module, AttentionLayerBase): kv_cache_scheme = getattr(quant_config, "kv_cache_scheme", None) if kv_cache_scheme is not None and kv_cache_dtype == "auto": kv_cache_dtype = "fp8" - calculate_kv_scales = False if cache_config is not None: cache_config.cache_dtype = "fp8" - cache_config.calculate_kv_scales = False # Check if per-head quant scales are required based on kv_cache_scheme use_per_head_quant_scales = ( @@ -312,7 +303,6 @@ class Attention(nn.Module, AttentionLayerBase): skip = True if skip: kv_cache_dtype = "auto" - calculate_kv_scales = False logger.debug( "Layer %s: kv_cache_dtype=%s, sliding_window=%s", prefix, @@ -324,7 +314,6 @@ class Attention(nn.Module, AttentionLayerBase): kv_cache_dtype, vllm_config.model_config ) self.kv_cache_dtype = kv_cache_dtype - self.calculate_kv_scales = calculate_kv_scales if num_kv_heads is None: num_kv_heads = num_heads assert num_heads % num_kv_heads == 0, ( @@ -505,10 +494,6 @@ class Attention(nn.Module, AttentionLayerBase): context using `vllm.forward_context.get_forward_context().attn_metadata`. """ - if self.calculate_kv_scales: - torch.ops.vllm.maybe_calc_kv_scales( - query, key, value, _encode_layer_name(self.layer_name) - ) if output_dtype is None: output_dtype = query.dtype if self.query_quant is not None: @@ -581,18 +566,6 @@ class Attention(nn.Module, AttentionLayerBase): ) return output.view(-1, hidden_size) - def calc_kv_scales(self, query, key, value): - self._q_scale.copy_(torch.abs(query).max() / self.q_range) - self._k_scale.copy_(torch.abs(key).max() / self.k_range) - self._v_scale.copy_(torch.abs(value).max() / self.v_range) - self._q_scale_float = self._q_scale.item() - self._k_scale_float = self._k_scale.item() - self._v_scale_float = self._v_scale.item() - self._k_scale_cpu.fill_(self._k_scale_float) - self._v_scale_cpu.fill_(self._v_scale_float) - # We only calculate the scales once - self.calculate_kv_scales = False - def extra_repr(self) -> str: s = f"head_size={self.impl.head_size}" # type: ignore s += f", num_heads={self.impl.num_heads}" # type: ignore @@ -694,41 +667,6 @@ class Attention(nn.Module, AttentionLayerBase): ) -def maybe_calc_kv_scales( - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - layer_name: LayerNameType, -) -> None: - layer_name = _resolve_layer_name(layer_name) - forward_context: ForwardContext = get_forward_context() - self = forward_context.no_compile_layers[layer_name] - - # Only calculate if the layer's calculate_kv_scales flag is True - # This flag gets set to False after the first forward pass - if not self.calculate_kv_scales: - return - - self.calc_kv_scales(query, key, value) - - -def maybe_calc_kv_scales_fake( - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - layer_name: LayerNameType, -) -> None: - return - - -direct_register_custom_op( - op_name="maybe_calc_kv_scales", - op_func=maybe_calc_kv_scales, - mutates_args=["query", "key", "value"], - fake_impl=maybe_calc_kv_scales_fake, -) - - def get_attention_context( layer_name: str, ) -> tuple[Any, "Attention | MLAAttention", torch.Tensor, torch.Tensor]: diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 3f737bed58..53b62da912 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -399,10 +399,8 @@ class MLAAttention(nn.Module, AttentionLayerBase): if cache_config is not None: kv_cache_dtype: CacheDType = cache_config.cache_dtype - calculate_kv_scales = cache_config.calculate_kv_scales else: kv_cache_dtype = "auto" - calculate_kv_scales = False self.quant_config = quant_config if cache_config is not None and cache_config.kv_cache_dtype_skip_layers: @@ -411,7 +409,6 @@ class MLAAttention(nn.Module, AttentionLayerBase): layer_idx = extract_layer_index(prefix) if str(layer_idx) in cache_config.kv_cache_dtype_skip_layers: kv_cache_dtype = "auto" - calculate_kv_scales = False logger.debug( "Layer %s: kv_cache_dtype=%s", prefix, @@ -460,7 +457,6 @@ class MLAAttention(nn.Module, AttentionLayerBase): # Initialize KV cache quantization attributes self.kv_cache_dtype = kv_cache_dtype - self.calculate_kv_scales = calculate_kv_scales _init_kv_cache_quant(self, quant_config, prefix) if ( @@ -560,11 +556,6 @@ class MLAAttention(nn.Module, AttentionLayerBase): and _vllm_config.parallel_config.dcp_comm_backend == "a2a" ) - # Initialize q/k/v range constants. - self.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32) - self.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32) - self.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32) - self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled() # If kv_b_proj_weight is unquantized, quantize it to mxfp4 if supported @@ -606,14 +597,6 @@ class MLAAttention(nn.Module, AttentionLayerBase): output_shape: torch.Size | None = None, q_dcp_replicated: torch.Tensor | None = None, ) -> torch.Tensor: - if self.calculate_kv_scales: - torch.ops.vllm.maybe_calc_kv_scales( - q, - kv_c_normed, - k_pe, - _encode_layer_name(self.layer_name), - ) - if self.use_direct_call: forward_context: ForwardContext = get_forward_context() attn_metadata_raw = forward_context.attn_metadata @@ -1110,30 +1093,6 @@ class MLAAttention(nn.Module, AttentionLayerBase): if not should_load_quant_weights(quant_method): set_default_quant_scales(self, register_buffer=False) - def calc_kv_scales( - self, q: torch.Tensor, kv_c_normed: torch.Tensor, k_pe: torch.Tensor - ) -> None: - """Optional scale calculation for MLA inputs. - - Mirrors Attention.calc_kv_scales. Not all MLA backends require this - """ - # Use safe defaults if ranges are not present - q_range = getattr(self, "q_range", torch.tensor(1.0)) - k_range = getattr(self, "k_range", torch.tensor(1.0)) - v_range = getattr(self, "v_range", torch.tensor(1.0)) - - self._q_scale.copy_(torch.abs(q).max() / q_range) - # kv_c_normed is the compressed KV representation; use it for k/v - kv_abs_max = torch.abs(kv_c_normed).max() - self._k_scale.copy_(kv_abs_max / k_range) - self._v_scale.copy_(kv_abs_max / v_range) - self._q_scale_float = self._q_scale.item() - self._k_scale_float = self._k_scale.item() - self._v_scale_float = self._v_scale.item() - self._k_scale_cpu.fill_(self._k_scale_float) - self._v_scale_cpu.fill_(self._v_scale_float) - self.calculate_kv_scales = False - def get_attn_backend(self) -> type[AttentionBackend]: return self.attn_backend diff --git a/vllm/model_executor/layers/quantization/kv_cache.py b/vllm/model_executor/layers/quantization/kv_cache.py index 380ad33f28..ee535bab2d 100644 --- a/vllm/model_executor/layers/quantization/kv_cache.py +++ b/vllm/model_executor/layers/quantization/kv_cache.py @@ -80,8 +80,7 @@ class BaseKVCacheMethod(QuantizeMethodBase): return # Per-token-head quantized KV cache: scales are computed dynamically - # per (token, head) in the kernel at cache-write time. Checkpoint - # scales are never used regardless of calculate_kv_scales. + # per (token, head) in the kernel at cache-write time. if kv_cache_uses_per_token_head_scales(layer.kv_cache_dtype): layer._k_scale.copy_(1.0) layer._v_scale.copy_(1.0) @@ -97,10 +96,7 @@ class BaseKVCacheMethod(QuantizeMethodBase): # regardless whether the kv-scale is available in the checkpoint. # No need to process kv scales after loading if we are going to # calculate them on the fly. - if ( - is_quantized_kv_cache(layer.kv_cache_dtype) - and not layer.calculate_kv_scales - ): + if is_quantized_kv_cache(layer.kv_cache_dtype): if layer.k_scale > 0.0 and layer.v_scale > 0.0: # We prefer to use separate k_scale and v_scale if present k_scale = layer.k_scale.to("cpu").tolist() @@ -158,7 +154,6 @@ class BaseKVCacheMethod(QuantizeMethodBase): q_scale = layer.q_scale if current_platform.is_fp8_fnuz(): q_scale *= 2 - layer.calculate_kv_scales = False else: q_scale = 1.0 if layer.prob_scale > 0.0: diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index 66427dfcdd..f4ffc24678 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -445,23 +445,6 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig): Args: vllm_config: vLLM Config """ - cache_config = vllm_config.cache_config - - # Disable calculate_kv_scales for hybrid models: uninitialized - # recurrent state corrupts scales during the calibration pass. - # See issue: https://github.com/vllm-project/vllm/issues/37554 - - if cache_config.calculate_kv_scales: - logger.warning( - "Disabling calculate_kv_scales for hybrid model '%s'. " - "Hybrid models with recurrent layers (GDN, Mamba, SSM) " - "produce unreliable KV cache scales during the " - "calibration pass because recurrent state is " - "uninitialized. Using default scale of 1.0 instead.", - vllm_config.model_config.model, - ) - cache_config.calculate_kv_scales = False - # Enable FULL_AND_PIECEWISE by default MambaModelConfig.verify_and_update_config(vllm_config) diff --git a/vllm/models/minimax_m3/amd/model.py b/vllm/models/minimax_m3/amd/model.py index 6960676666..6aa172f141 100644 --- a/vllm/models/minimax_m3/amd/model.py +++ b/vllm/models/minimax_m3/amd/model.py @@ -642,7 +642,6 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): # MiniMax-M3 sparse attention owns its KV-cache insert/read path instead # of wrapping the generic Attention module. Keep the same runtime scale # attributes so FP8 KV reads can honor vLLM's per-layer descale contract. - self.calculate_kv_scales = False set_default_quant_scales(self, register_buffer=True) # Shared top-k buffer: the indexer writes the selected blocks into it and diff --git a/vllm/models/minimax_m3/nvidia/model.py b/vllm/models/minimax_m3/nvidia/model.py index 03e28f3443..68f9805b1f 100644 --- a/vllm/models/minimax_m3/nvidia/model.py +++ b/vllm/models/minimax_m3/nvidia/model.py @@ -495,7 +495,6 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): # MiniMax-M3 sparse attention owns its KV-cache insert/read path instead # of wrapping the generic Attention module. Keep the same runtime scale # attributes so FP8 KV reads can honor vLLM's per-layer descale contract. - self.calculate_kv_scales = False set_default_quant_scales(self, register_buffer=True) # Indexer side-cache dtype, mirroring --kv-cache-dtype for the main # cache (--attention-config '{"indexer_kv_dtype": ...}'). diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 5083087bf0..5f6ce2bc76 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -499,7 +499,6 @@ class GPUModelRunner( self.max_model_len = model_config.max_model_len # Always set to false after the first forward pass - self.calculate_kv_scales = self.cache_config.calculate_kv_scales self.dcp_world_size = self.parallel_config.decode_context_parallel_size self.dcp_rank = 0 if self.dcp_world_size <= 1 else get_dcp_group().rank_in_group self.max_num_tokens = scheduler_config.max_num_batched_tokens @@ -4402,14 +4401,6 @@ class GPUModelRunner( scheduler_output, num_tokens_padded, intermediate_tensors ) - # Set cudagraph mode to none if calc_kv_scales is true. - # KV scales calculation involves dynamic operations that are incompatible - # with CUDA graph capture. - if self.calculate_kv_scales: - cudagraph_mode = CUDAGraphMode.NONE - # Mark KV scales as calculated after the first forward pass - self.calculate_kv_scales = False - # Encoder-decoder models can only compile the pure decode steps where no # encoder inputs are present. Use eager for the first pass. num_encoder_reqs = len(scheduler_output.scheduled_encoder_inputs)