[Misc] Remove deprecated calculate_kv_scales runtime KV scale calculation (#49389)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
This commit is contained in:
wangxiyuan
2026-08-03 11:15:13 -04:00
committed by GitHub
parent 0cf49a5d15
commit dd11df04f3
19 changed files with 16 additions and 263 deletions
+1 -1
View File
@@ -636,7 +636,7 @@ steps:
- tests/compile
- vllm/platforms/rocm.py
commands:
- pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile'
- pytest -v -s compile/fullgraph/test_full_graph.py
- label: Pytorch Nightly Dependency Override Check # TBD
timeout_in_minutes: 180
-3
View File
@@ -90,9 +90,6 @@ steps:
- pytest -v -s tests/compile/passes/test_silu_mul_quant_fusion.py
# this runner has 2 GPUs available even though num_devices=2 is not set
- pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py
# test_fp8_kv_scale_compile requires FlashAttention (not supported on default L4/L40)
# TODO(luka) move to H100 once pass tests run on H100
- pytest -v -s tests/compile/fullgraph/test_full_graph.py::test_fp8_kv_scale_compile
- label: Fusion E2E Quick (H100)
key: fusion-e2e-quick-h100
+1 -2
View File
@@ -211,8 +211,7 @@ steps:
- vllm/v1/
- tests/compile
commands:
# fp8 kv scales not supported on sm89, tested on Blackwell instead
- pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile'
- pytest -v -s compile/fullgraph/test_full_graph.py
- label: Pytorch Nightly Dependency Override Check # 2min
key: pytorch-nightly-dependency-override-check
+1 -1
View File
@@ -367,7 +367,7 @@ We use this concept for the `vllm:cache_config_info` metric:
```text
# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig
# TYPE vllm:cache_config_info gauge
vllm:cache_config_info{block_size="16",cache_dtype="auto",calculate_kv_scales="False",cpu_offload_gb="0",enable_prefix_caching="False",gpu_memory_utilization="0.9",...} 1.0
vllm:cache_config_info{block_size="16",cache_dtype="auto",cpu_offload_gb="0",enable_prefix_caching="False",gpu_memory_utilization="0.9",...} 1.0
```
However, `prometheus_client` has
@@ -27,18 +27,9 @@ You can configure how the quantization scales are computed in vLLM using three d
_Configure with:_
```python
kv_cache_dtype="fp8"
calculate_kv_scales=False
```
2. **Random token calibration (on-the-fly):**
Scales are automatically estimated from a single batch of random tokens during warmup and then fixed.
_Configure with:_
```python
kv_cache_dtype="fp8"
calculate_kv_scales=True
```
3. **[Recommended] Calibration with a dataset (via llm-compressor):**
2. **[Recommended] Calibration with a dataset (via llm-compressor):**
Scales are estimated using a curated calibration dataset for maximum accuracy.
This requires the [llm-compressor](https://github.com/vllm-project/llm-compressor) library.
_See example below!_
@@ -79,7 +70,7 @@ llm = LLM(
## Examples
### 1. No Calibration (`kv_cache_dtype="fp8"`, `calculate_kv_scales=False`)
### 1. No Calibration (`kv_cache_dtype="fp8"`)
All quantization scales are set to 1.0.
@@ -90,7 +81,6 @@ sampling_params = SamplingParams(temperature=0.7, top_p=0.8)
llm = LLM(
model="meta-llama/Llama-2-7b-chat-hf",
kv_cache_dtype="fp8",
calculate_kv_scales=False,
)
prompt = "London is the capital of"
out = llm.generate(prompt, sampling_params)[0].outputs[0].text
@@ -99,27 +89,7 @@ print(out)
---
### 2. Random Token Calibration (`kv_cache_dtype="fp8"`, `calculate_kv_scales=True`)
Scales are automatically estimated from a single batch of tokens during warmup.
```python
from vllm import LLM, SamplingParams
sampling_params = SamplingParams(temperature=0.7, top_p=0.8)
llm = LLM(
model="meta-llama/Llama-2-7b-chat-hf",
kv_cache_dtype="fp8",
calculate_kv_scales=True,
)
prompt = "London is the capital of"
out = llm.generate(prompt, sampling_params)[0].outputs[0].text
print(out)
```
---
### 3. **[Recommended] Calibration Using a Dataset (with `llm-compressor`)**
### 2. **[Recommended] Calibration Using a Dataset (with `llm-compressor`)**
For the highest-quality quantization, we recommend calibrating against a dataset using `llm-compressor`. This enables advanced strategies such as per-attention-head quantization.
@@ -13,7 +13,6 @@ from vllm import LLM, SamplingParams
from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode, PassConfig
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_torch_equal_or_newer
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from ...utils import create_new_process_for_each_test
@@ -182,37 +181,6 @@ def test_custom_compile_config(
run_model(compilation_config, model, **model_kwargs)
@pytest.mark.parametrize(
"compilation_mode",
[CompilationMode.NONE, CompilationMode.VLLM_COMPILE],
)
@pytest.mark.parametrize(
"model, backend",
[
("Qwen/Qwen2-0.5B", None), # Standard attention model
(
"deepseek-ai/DeepSeek-V2-Lite",
AttentionBackendEnum.FLASHINFER_MLA,
), # MLA (Multi-head Latent Attention) model
],
)
def test_fp8_kv_scale_compile(
compilation_mode: CompilationMode,
model: str,
backend: AttentionBackendEnum | None,
):
model_kwargs = {
"quantization": "fp8",
"kv_cache_dtype": "fp8_e4m3",
"calculate_kv_scales": True,
"max_model_len": 512,
}
if backend:
model_kwargs["attention_config"] = {"backend": backend.name}
run_model(compilation_mode, model, **model_kwargs)
def run_model(
compile_config: CompilationMode | CompilationConfig, model: str, **model_kwargs
):
@@ -52,9 +52,6 @@ def test_per_token_head_kv_cache_accuracy(
) -> None:
"""Compare logprobs between bf16 baseline and per-token-head quantized KV
cache.
Uses calculate_kv_scales (dynamic scale computation) since there are
no per-token-head calibrated checkpoints available yet.
"""
with monkeypatch.context() as m:
m.setenv("TOKENIZERS_PARALLELISM", "true")
@@ -80,7 +77,6 @@ def test_per_token_head_kv_cache_accuracy(
tensor_parallel_size=tensor_parallel_size,
enforce_eager=enforce_eager,
kv_cache_dtype=kv_cache_dtype,
calculate_kv_scales=True,
attention_config={"backend": backend},
) as vllm_model:
test_outputs = vllm_model.generate_greedy_logprobs(
+8 -20
View File
@@ -18,7 +18,6 @@ from vllm.model_executor.kernels.linear.scaled_mm import (
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.attention.attention import (
Attention,
set_default_quant_scales,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
@@ -433,8 +432,7 @@ def test_fp8_reloading(
method.process_weights_after_loading(layer)
@pytest.mark.parametrize("source", ["checkpoint", "runtime_calc"])
def test_kv_cache_scale_sync_to_host_copies(source):
def test_kv_cache_scale_sync_to_host_copies():
"""Test device-to-host sync of the k/v quantization scales, for both the
checkpoint-load and runtime-calc paths that produce them.
"""
@@ -442,23 +440,13 @@ def test_kv_cache_scale_sync_to_host_copies(source):
set_default_quant_scales(layer, register_buffer=True)
layer.kv_cache_dtype = "fp8"
if source == "checkpoint":
# Scales come from the checkpoint, so runtime calc is disabled.
layer.calculate_kv_scales = False
method = BaseKVCacheMethod(quant_config=None)
method.create_weights(layer)
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
layer.k_scale.weight_loader(layer.k_scale, checkpoint_scale)
layer.v_scale.weight_loader(layer.v_scale, checkpoint_scale)
method.process_weights_after_loading(layer)
else:
# First forward computes distinct, non-unity scales from live k/v.
layer.calculate_kv_scales = True
query = torch.full((4, 8), 10.0)
key = torch.full((4, 8), 60.0)
value = torch.full((4, 8), 50.0)
Attention.calc_kv_scales(layer, query, key, value)
method = BaseKVCacheMethod(quant_config=None)
method.create_weights(layer)
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
layer.k_scale.weight_loader(layer.k_scale, checkpoint_scale)
layer.v_scale.weight_loader(layer.v_scale, checkpoint_scale)
method.process_weights_after_loading(layer)
assert layer._k_scale_float != 1.0
assert layer._v_scale_float != 1.0
@@ -497,7 +497,6 @@ def test_process_weights_sets_placeholder_scales(kv_cache_dtype: str):
layer = MagicMock()
layer.kv_cache_dtype = kv_cache_dtype
layer.calculate_kv_scales = False
layer.k_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer.v_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
layer.q_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
-17
View File
@@ -108,11 +108,6 @@ class CacheConfig:
security risk tolerance against the performance benefits before turning this on.
- "xxhash_cbor" combines canonical CBOR serialization with xxHash for
reproducible hashing. Requires the optional ``xxhash`` package."""
calculate_kv_scales: bool = False
"""Deprecated: This option is deprecated and will be removed in v0.19.
It enables dynamic calculation of `k_scale` and `v_scale` when
kv_cache_dtype is fp8. If `False`, the scales will be loaded from the model
checkpoint if available. Otherwise, the scales will default to 1.0."""
kv_cache_dtype_skip_layers: list[str] = field(default_factory=list)
"""Layer patterns to skip KV cache quantization. Accepts layer indices
(e.g., '0', '2', '4') or attention type names (e.g., 'sliding_window')."""
@@ -270,18 +265,6 @@ class CacheConfig:
self.user_specified_mamba_block_size = True
return self
@field_validator("calculate_kv_scales", mode="after")
@classmethod
def _warn_deprecated_calculate_kv_scales(cls, calculate_kv_scales: bool) -> bool:
if calculate_kv_scales:
logger.warning(
"The `--calculate-kv-scales` option is deprecated and will "
"be removed in v0.19. The scales will be loaded from the "
"model checkpoint if available, otherwise they default to "
"1.0."
)
return calculate_kv_scales
@field_validator("cache_dtype", mode="after")
@classmethod
def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType:
-5
View File
@@ -695,7 +695,6 @@ class EngineArgs:
override_attention_dtype: str | None = ModelConfig.override_attention_dtype
attention_backend: AttentionBackendEnum | None = AttentionConfig.backend
calculate_kv_scales: bool = CacheConfig.calculate_kv_scales
kv_cache_dtype_skip_layers: list[str] = get_field(
CacheConfig, "kv_cache_dtype_skip_layers"
)
@@ -1210,9 +1209,6 @@ class EngineArgs:
cache_group.add_argument(
"--prefix-caching-hash-algo", **cache_kwargs["prefix_caching_hash_algo"]
)
cache_group.add_argument(
"--calculate-kv-scales", **cache_kwargs["calculate_kv_scales"]
)
cache_group.add_argument(
"--kv-cache-dtype-skip-layers", **cache_kwargs["kv_cache_dtype_skip_layers"]
)
@@ -1957,7 +1953,6 @@ class EngineArgs:
sliding_window=sliding_window,
enable_prefix_caching=self.enable_prefix_caching,
prefix_caching_hash_algo=self.prefix_caching_hash_algo,
calculate_kv_scales=self.calculate_kv_scales,
kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers,
kv_sharing_fast_prefill=self.kv_sharing_fast_prefill,
mamba_cache_dtype=self.mamba_cache_dtype,
-6
View File
@@ -1325,12 +1325,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB": lambda: maybe_convert_int(
os.environ.get("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB", None)
),
# Divisor for dynamic query scale factor calculation for FP8 KV Cache
"Q_SCALE_CONSTANT": lambda: int(os.getenv("Q_SCALE_CONSTANT", "200")),
# Divisor for dynamic key scale factor calculation for FP8 KV Cache
"K_SCALE_CONSTANT": lambda: int(os.getenv("K_SCALE_CONSTANT", "200")),
# Divisor for dynamic value scale factor calculation for FP8 KV Cache
"V_SCALE_CONSTANT": lambda: int(os.getenv("V_SCALE_CONSTANT", "100")),
# If set, enable multiprocessing in LLM for the V1 code path.
"VLLM_ENABLE_V1_MULTIPROCESSING": lambda: bool(
int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1"))
@@ -144,11 +144,6 @@ def set_default_quant_scales(layer: nn.Module, register_buffer: bool = False) ->
layer._v_scale_cpu = torch.tensor(1.0, dtype=torch.float32)
layer._prob_scale_float = 1.0
# Initialize q/k/v range constants used by calc_kv_scales
layer.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32)
layer.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
layer.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
def _init_kv_cache_quant(
layer: nn.Module,
@@ -270,10 +265,8 @@ class Attention(nn.Module, AttentionLayerBase):
vllm_config = get_current_vllm_config()
if cache_config is not None:
kv_cache_dtype = cache_config.cache_dtype
calculate_kv_scales = cache_config.calculate_kv_scales
else:
kv_cache_dtype = "auto"
calculate_kv_scales = False
# llm-compressor models declare an FP8 KV-cache scheme in their
# checkpoint config. Honor it only when the user did not explicitly
@@ -284,10 +277,8 @@ class Attention(nn.Module, AttentionLayerBase):
kv_cache_scheme = getattr(quant_config, "kv_cache_scheme", None)
if kv_cache_scheme is not None and kv_cache_dtype == "auto":
kv_cache_dtype = "fp8"
calculate_kv_scales = False
if cache_config is not None:
cache_config.cache_dtype = "fp8"
cache_config.calculate_kv_scales = False
# Check if per-head quant scales are required based on kv_cache_scheme
use_per_head_quant_scales = (
@@ -312,7 +303,6 @@ class Attention(nn.Module, AttentionLayerBase):
skip = True
if skip:
kv_cache_dtype = "auto"
calculate_kv_scales = False
logger.debug(
"Layer %s: kv_cache_dtype=%s, sliding_window=%s",
prefix,
@@ -324,7 +314,6 @@ class Attention(nn.Module, AttentionLayerBase):
kv_cache_dtype, vllm_config.model_config
)
self.kv_cache_dtype = kv_cache_dtype
self.calculate_kv_scales = calculate_kv_scales
if num_kv_heads is None:
num_kv_heads = num_heads
assert num_heads % num_kv_heads == 0, (
@@ -505,10 +494,6 @@ class Attention(nn.Module, AttentionLayerBase):
context using
`vllm.forward_context.get_forward_context().attn_metadata`.
"""
if self.calculate_kv_scales:
torch.ops.vllm.maybe_calc_kv_scales(
query, key, value, _encode_layer_name(self.layer_name)
)
if output_dtype is None:
output_dtype = query.dtype
if self.query_quant is not None:
@@ -581,18 +566,6 @@ class Attention(nn.Module, AttentionLayerBase):
)
return output.view(-1, hidden_size)
def calc_kv_scales(self, query, key, value):
self._q_scale.copy_(torch.abs(query).max() / self.q_range)
self._k_scale.copy_(torch.abs(key).max() / self.k_range)
self._v_scale.copy_(torch.abs(value).max() / self.v_range)
self._q_scale_float = self._q_scale.item()
self._k_scale_float = self._k_scale.item()
self._v_scale_float = self._v_scale.item()
self._k_scale_cpu.fill_(self._k_scale_float)
self._v_scale_cpu.fill_(self._v_scale_float)
# We only calculate the scales once
self.calculate_kv_scales = False
def extra_repr(self) -> str:
s = f"head_size={self.impl.head_size}" # type: ignore
s += f", num_heads={self.impl.num_heads}" # type: ignore
@@ -694,41 +667,6 @@ class Attention(nn.Module, AttentionLayerBase):
)
def maybe_calc_kv_scales(
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
layer_name: LayerNameType,
) -> None:
layer_name = _resolve_layer_name(layer_name)
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
# Only calculate if the layer's calculate_kv_scales flag is True
# This flag gets set to False after the first forward pass
if not self.calculate_kv_scales:
return
self.calc_kv_scales(query, key, value)
def maybe_calc_kv_scales_fake(
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
layer_name: LayerNameType,
) -> None:
return
direct_register_custom_op(
op_name="maybe_calc_kv_scales",
op_func=maybe_calc_kv_scales,
mutates_args=["query", "key", "value"],
fake_impl=maybe_calc_kv_scales_fake,
)
def get_attention_context(
layer_name: str,
) -> tuple[Any, "Attention | MLAAttention", torch.Tensor, torch.Tensor]:
@@ -399,10 +399,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
if cache_config is not None:
kv_cache_dtype: CacheDType = cache_config.cache_dtype
calculate_kv_scales = cache_config.calculate_kv_scales
else:
kv_cache_dtype = "auto"
calculate_kv_scales = False
self.quant_config = quant_config
if cache_config is not None and cache_config.kv_cache_dtype_skip_layers:
@@ -411,7 +409,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
layer_idx = extract_layer_index(prefix)
if str(layer_idx) in cache_config.kv_cache_dtype_skip_layers:
kv_cache_dtype = "auto"
calculate_kv_scales = False
logger.debug(
"Layer %s: kv_cache_dtype=%s",
prefix,
@@ -460,7 +457,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
# Initialize KV cache quantization attributes
self.kv_cache_dtype = kv_cache_dtype
self.calculate_kv_scales = calculate_kv_scales
_init_kv_cache_quant(self, quant_config, prefix)
if (
@@ -560,11 +556,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
and _vllm_config.parallel_config.dcp_comm_backend == "a2a"
)
# Initialize q/k/v range constants.
self.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32)
self.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
self.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
# If kv_b_proj_weight is unquantized, quantize it to mxfp4 if supported
@@ -606,14 +597,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
output_shape: torch.Size | None = None,
q_dcp_replicated: torch.Tensor | None = None,
) -> torch.Tensor:
if self.calculate_kv_scales:
torch.ops.vllm.maybe_calc_kv_scales(
q,
kv_c_normed,
k_pe,
_encode_layer_name(self.layer_name),
)
if self.use_direct_call:
forward_context: ForwardContext = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
@@ -1110,30 +1093,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
if not should_load_quant_weights(quant_method):
set_default_quant_scales(self, register_buffer=False)
def calc_kv_scales(
self, q: torch.Tensor, kv_c_normed: torch.Tensor, k_pe: torch.Tensor
) -> None:
"""Optional scale calculation for MLA inputs.
Mirrors Attention.calc_kv_scales. Not all MLA backends require this
"""
# Use safe defaults if ranges are not present
q_range = getattr(self, "q_range", torch.tensor(1.0))
k_range = getattr(self, "k_range", torch.tensor(1.0))
v_range = getattr(self, "v_range", torch.tensor(1.0))
self._q_scale.copy_(torch.abs(q).max() / q_range)
# kv_c_normed is the compressed KV representation; use it for k/v
kv_abs_max = torch.abs(kv_c_normed).max()
self._k_scale.copy_(kv_abs_max / k_range)
self._v_scale.copy_(kv_abs_max / v_range)
self._q_scale_float = self._q_scale.item()
self._k_scale_float = self._k_scale.item()
self._v_scale_float = self._v_scale.item()
self._k_scale_cpu.fill_(self._k_scale_float)
self._v_scale_cpu.fill_(self._v_scale_float)
self.calculate_kv_scales = False
def get_attn_backend(self) -> type[AttentionBackend]:
return self.attn_backend
@@ -80,8 +80,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
return
# Per-token-head quantized KV cache: scales are computed dynamically
# per (token, head) in the kernel at cache-write time. Checkpoint
# scales are never used regardless of calculate_kv_scales.
# per (token, head) in the kernel at cache-write time.
if kv_cache_uses_per_token_head_scales(layer.kv_cache_dtype):
layer._k_scale.copy_(1.0)
layer._v_scale.copy_(1.0)
@@ -97,10 +96,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
# regardless whether the kv-scale is available in the checkpoint.
# No need to process kv scales after loading if we are going to
# calculate them on the fly.
if (
is_quantized_kv_cache(layer.kv_cache_dtype)
and not layer.calculate_kv_scales
):
if is_quantized_kv_cache(layer.kv_cache_dtype):
if layer.k_scale > 0.0 and layer.v_scale > 0.0:
# We prefer to use separate k_scale and v_scale if present
k_scale = layer.k_scale.to("cpu").tolist()
@@ -158,7 +154,6 @@ class BaseKVCacheMethod(QuantizeMethodBase):
q_scale = layer.q_scale
if current_platform.is_fp8_fnuz():
q_scale *= 2
layer.calculate_kv_scales = False
else:
q_scale = 1.0
if layer.prob_scale > 0.0:
-17
View File
@@ -445,23 +445,6 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
Args:
vllm_config: vLLM Config
"""
cache_config = vllm_config.cache_config
# Disable calculate_kv_scales for hybrid models: uninitialized
# recurrent state corrupts scales during the calibration pass.
# See issue: https://github.com/vllm-project/vllm/issues/37554
if cache_config.calculate_kv_scales:
logger.warning(
"Disabling calculate_kv_scales for hybrid model '%s'. "
"Hybrid models with recurrent layers (GDN, Mamba, SSM) "
"produce unreliable KV cache scales during the "
"calibration pass because recurrent state is "
"uninitialized. Using default scale of 1.0 instead.",
vllm_config.model_config.model,
)
cache_config.calculate_kv_scales = False
# Enable FULL_AND_PIECEWISE by default
MambaModelConfig.verify_and_update_config(vllm_config)
-1
View File
@@ -642,7 +642,6 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
# MiniMax-M3 sparse attention owns its KV-cache insert/read path instead
# of wrapping the generic Attention module. Keep the same runtime scale
# attributes so FP8 KV reads can honor vLLM's per-layer descale contract.
self.calculate_kv_scales = False
set_default_quant_scales(self, register_buffer=True)
# Shared top-k buffer: the indexer writes the selected blocks into it and
-1
View File
@@ -495,7 +495,6 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
# MiniMax-M3 sparse attention owns its KV-cache insert/read path instead
# of wrapping the generic Attention module. Keep the same runtime scale
# attributes so FP8 KV reads can honor vLLM's per-layer descale contract.
self.calculate_kv_scales = False
set_default_quant_scales(self, register_buffer=True)
# Indexer side-cache dtype, mirroring --kv-cache-dtype for the main
# cache (--attention-config '{"indexer_kv_dtype": ...}').
-9
View File
@@ -499,7 +499,6 @@ class GPUModelRunner(
self.max_model_len = model_config.max_model_len
# Always set to false after the first forward pass
self.calculate_kv_scales = self.cache_config.calculate_kv_scales
self.dcp_world_size = self.parallel_config.decode_context_parallel_size
self.dcp_rank = 0 if self.dcp_world_size <= 1 else get_dcp_group().rank_in_group
self.max_num_tokens = scheduler_config.max_num_batched_tokens
@@ -4402,14 +4401,6 @@ class GPUModelRunner(
scheduler_output, num_tokens_padded, intermediate_tensors
)
# Set cudagraph mode to none if calc_kv_scales is true.
# KV scales calculation involves dynamic operations that are incompatible
# with CUDA graph capture.
if self.calculate_kv_scales:
cudagraph_mode = CUDAGraphMode.NONE
# Mark KV scales as calculated after the first forward pass
self.calculate_kv_scales = False
# Encoder-decoder models can only compile the pure decode steps where no
# encoder inputs are present. Use eager for the first pass.
num_encoder_reqs = len(scheduler_output.scheduled_encoder_inputs)