[Misc] Remove deprecated calculate_kv_scales runtime KV scale calculation (#49389)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
This commit is contained in:
@@ -636,7 +636,7 @@ steps:
|
||||
- tests/compile
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile'
|
||||
- pytest -v -s compile/fullgraph/test_full_graph.py
|
||||
|
||||
- label: Pytorch Nightly Dependency Override Check # TBD
|
||||
timeout_in_minutes: 180
|
||||
|
||||
@@ -90,9 +90,6 @@ steps:
|
||||
- pytest -v -s tests/compile/passes/test_silu_mul_quant_fusion.py
|
||||
# this runner has 2 GPUs available even though num_devices=2 is not set
|
||||
- pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py
|
||||
# test_fp8_kv_scale_compile requires FlashAttention (not supported on default L4/L40)
|
||||
# TODO(luka) move to H100 once pass tests run on H100
|
||||
- pytest -v -s tests/compile/fullgraph/test_full_graph.py::test_fp8_kv_scale_compile
|
||||
|
||||
- label: Fusion E2E Quick (H100)
|
||||
key: fusion-e2e-quick-h100
|
||||
|
||||
@@ -211,8 +211,7 @@ steps:
|
||||
- vllm/v1/
|
||||
- tests/compile
|
||||
commands:
|
||||
# fp8 kv scales not supported on sm89, tested on Blackwell instead
|
||||
- pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile'
|
||||
- pytest -v -s compile/fullgraph/test_full_graph.py
|
||||
|
||||
- label: Pytorch Nightly Dependency Override Check # 2min
|
||||
key: pytorch-nightly-dependency-override-check
|
||||
|
||||
@@ -367,7 +367,7 @@ We use this concept for the `vllm:cache_config_info` metric:
|
||||
```text
|
||||
# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig
|
||||
# TYPE vllm:cache_config_info gauge
|
||||
vllm:cache_config_info{block_size="16",cache_dtype="auto",calculate_kv_scales="False",cpu_offload_gb="0",enable_prefix_caching="False",gpu_memory_utilization="0.9",...} 1.0
|
||||
vllm:cache_config_info{block_size="16",cache_dtype="auto",cpu_offload_gb="0",enable_prefix_caching="False",gpu_memory_utilization="0.9",...} 1.0
|
||||
```
|
||||
|
||||
However, `prometheus_client` has
|
||||
|
||||
@@ -27,18 +27,9 @@ You can configure how the quantization scales are computed in vLLM using three d
|
||||
_Configure with:_
|
||||
```python
|
||||
kv_cache_dtype="fp8"
|
||||
calculate_kv_scales=False
|
||||
```
|
||||
|
||||
2. **Random token calibration (on-the-fly):**
|
||||
Scales are automatically estimated from a single batch of random tokens during warmup and then fixed.
|
||||
_Configure with:_
|
||||
```python
|
||||
kv_cache_dtype="fp8"
|
||||
calculate_kv_scales=True
|
||||
```
|
||||
|
||||
3. **[Recommended] Calibration with a dataset (via llm-compressor):**
|
||||
2. **[Recommended] Calibration with a dataset (via llm-compressor):**
|
||||
Scales are estimated using a curated calibration dataset for maximum accuracy.
|
||||
This requires the [llm-compressor](https://github.com/vllm-project/llm-compressor) library.
|
||||
_See example below!_
|
||||
@@ -79,7 +70,7 @@ llm = LLM(
|
||||
|
||||
## Examples
|
||||
|
||||
### 1. No Calibration (`kv_cache_dtype="fp8"`, `calculate_kv_scales=False`)
|
||||
### 1. No Calibration (`kv_cache_dtype="fp8"`)
|
||||
|
||||
All quantization scales are set to 1.0.
|
||||
|
||||
@@ -90,7 +81,6 @@ sampling_params = SamplingParams(temperature=0.7, top_p=0.8)
|
||||
llm = LLM(
|
||||
model="meta-llama/Llama-2-7b-chat-hf",
|
||||
kv_cache_dtype="fp8",
|
||||
calculate_kv_scales=False,
|
||||
)
|
||||
prompt = "London is the capital of"
|
||||
out = llm.generate(prompt, sampling_params)[0].outputs[0].text
|
||||
@@ -99,27 +89,7 @@ print(out)
|
||||
|
||||
---
|
||||
|
||||
### 2. Random Token Calibration (`kv_cache_dtype="fp8"`, `calculate_kv_scales=True`)
|
||||
|
||||
Scales are automatically estimated from a single batch of tokens during warmup.
|
||||
|
||||
```python
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
sampling_params = SamplingParams(temperature=0.7, top_p=0.8)
|
||||
llm = LLM(
|
||||
model="meta-llama/Llama-2-7b-chat-hf",
|
||||
kv_cache_dtype="fp8",
|
||||
calculate_kv_scales=True,
|
||||
)
|
||||
prompt = "London is the capital of"
|
||||
out = llm.generate(prompt, sampling_params)[0].outputs[0].text
|
||||
print(out)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 3. **[Recommended] Calibration Using a Dataset (with `llm-compressor`)**
|
||||
### 2. **[Recommended] Calibration Using a Dataset (with `llm-compressor`)**
|
||||
|
||||
For the highest-quality quantization, we recommend calibrating against a dataset using `llm-compressor`. This enables advanced strategies such as per-attention-head quantization.
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ from vllm import LLM, SamplingParams
|
||||
from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode, PassConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from ...utils import create_new_process_for_each_test
|
||||
|
||||
@@ -182,37 +181,6 @@ def test_custom_compile_config(
|
||||
run_model(compilation_config, model, **model_kwargs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"compilation_mode",
|
||||
[CompilationMode.NONE, CompilationMode.VLLM_COMPILE],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"model, backend",
|
||||
[
|
||||
("Qwen/Qwen2-0.5B", None), # Standard attention model
|
||||
(
|
||||
"deepseek-ai/DeepSeek-V2-Lite",
|
||||
AttentionBackendEnum.FLASHINFER_MLA,
|
||||
), # MLA (Multi-head Latent Attention) model
|
||||
],
|
||||
)
|
||||
def test_fp8_kv_scale_compile(
|
||||
compilation_mode: CompilationMode,
|
||||
model: str,
|
||||
backend: AttentionBackendEnum | None,
|
||||
):
|
||||
model_kwargs = {
|
||||
"quantization": "fp8",
|
||||
"kv_cache_dtype": "fp8_e4m3",
|
||||
"calculate_kv_scales": True,
|
||||
"max_model_len": 512,
|
||||
}
|
||||
if backend:
|
||||
model_kwargs["attention_config"] = {"backend": backend.name}
|
||||
|
||||
run_model(compilation_mode, model, **model_kwargs)
|
||||
|
||||
|
||||
def run_model(
|
||||
compile_config: CompilationMode | CompilationConfig, model: str, **model_kwargs
|
||||
):
|
||||
|
||||
@@ -52,9 +52,6 @@ def test_per_token_head_kv_cache_accuracy(
|
||||
) -> None:
|
||||
"""Compare logprobs between bf16 baseline and per-token-head quantized KV
|
||||
cache.
|
||||
|
||||
Uses calculate_kv_scales (dynamic scale computation) since there are
|
||||
no per-token-head calibrated checkpoints available yet.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
@@ -80,7 +77,6 @@ def test_per_token_head_kv_cache_accuracy(
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enforce_eager=enforce_eager,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
calculate_kv_scales=True,
|
||||
attention_config={"backend": backend},
|
||||
) as vllm_model:
|
||||
test_outputs = vllm_model.generate_greedy_logprobs(
|
||||
|
||||
@@ -18,7 +18,6 @@ from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention.attention import (
|
||||
Attention,
|
||||
set_default_quant_scales,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
@@ -433,8 +432,7 @@ def test_fp8_reloading(
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["checkpoint", "runtime_calc"])
|
||||
def test_kv_cache_scale_sync_to_host_copies(source):
|
||||
def test_kv_cache_scale_sync_to_host_copies():
|
||||
"""Test device-to-host sync of the k/v quantization scales, for both the
|
||||
checkpoint-load and runtime-calc paths that produce them.
|
||||
"""
|
||||
@@ -442,23 +440,13 @@ def test_kv_cache_scale_sync_to_host_copies(source):
|
||||
set_default_quant_scales(layer, register_buffer=True)
|
||||
layer.kv_cache_dtype = "fp8"
|
||||
|
||||
if source == "checkpoint":
|
||||
# Scales come from the checkpoint, so runtime calc is disabled.
|
||||
layer.calculate_kv_scales = False
|
||||
method = BaseKVCacheMethod(quant_config=None)
|
||||
method.create_weights(layer)
|
||||
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
|
||||
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
|
||||
layer.k_scale.weight_loader(layer.k_scale, checkpoint_scale)
|
||||
layer.v_scale.weight_loader(layer.v_scale, checkpoint_scale)
|
||||
method.process_weights_after_loading(layer)
|
||||
else:
|
||||
# First forward computes distinct, non-unity scales from live k/v.
|
||||
layer.calculate_kv_scales = True
|
||||
query = torch.full((4, 8), 10.0)
|
||||
key = torch.full((4, 8), 60.0)
|
||||
value = torch.full((4, 8), 50.0)
|
||||
Attention.calc_kv_scales(layer, query, key, value)
|
||||
method = BaseKVCacheMethod(quant_config=None)
|
||||
method.create_weights(layer)
|
||||
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
|
||||
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
|
||||
layer.k_scale.weight_loader(layer.k_scale, checkpoint_scale)
|
||||
layer.v_scale.weight_loader(layer.v_scale, checkpoint_scale)
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
assert layer._k_scale_float != 1.0
|
||||
assert layer._v_scale_float != 1.0
|
||||
|
||||
@@ -497,7 +497,6 @@ def test_process_weights_sets_placeholder_scales(kv_cache_dtype: str):
|
||||
|
||||
layer = MagicMock()
|
||||
layer.kv_cache_dtype = kv_cache_dtype
|
||||
layer.calculate_kv_scales = False
|
||||
layer.k_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
|
||||
layer.v_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
|
||||
layer.q_scale = torch.nn.Parameter(torch.tensor(-1.0), requires_grad=False)
|
||||
|
||||
@@ -108,11 +108,6 @@ class CacheConfig:
|
||||
security risk tolerance against the performance benefits before turning this on.
|
||||
- "xxhash_cbor" combines canonical CBOR serialization with xxHash for
|
||||
reproducible hashing. Requires the optional ``xxhash`` package."""
|
||||
calculate_kv_scales: bool = False
|
||||
"""Deprecated: This option is deprecated and will be removed in v0.19.
|
||||
It enables dynamic calculation of `k_scale` and `v_scale` when
|
||||
kv_cache_dtype is fp8. If `False`, the scales will be loaded from the model
|
||||
checkpoint if available. Otherwise, the scales will default to 1.0."""
|
||||
kv_cache_dtype_skip_layers: list[str] = field(default_factory=list)
|
||||
"""Layer patterns to skip KV cache quantization. Accepts layer indices
|
||||
(e.g., '0', '2', '4') or attention type names (e.g., 'sliding_window')."""
|
||||
@@ -270,18 +265,6 @@ class CacheConfig:
|
||||
self.user_specified_mamba_block_size = True
|
||||
return self
|
||||
|
||||
@field_validator("calculate_kv_scales", mode="after")
|
||||
@classmethod
|
||||
def _warn_deprecated_calculate_kv_scales(cls, calculate_kv_scales: bool) -> bool:
|
||||
if calculate_kv_scales:
|
||||
logger.warning(
|
||||
"The `--calculate-kv-scales` option is deprecated and will "
|
||||
"be removed in v0.19. The scales will be loaded from the "
|
||||
"model checkpoint if available, otherwise they default to "
|
||||
"1.0."
|
||||
)
|
||||
return calculate_kv_scales
|
||||
|
||||
@field_validator("cache_dtype", mode="after")
|
||||
@classmethod
|
||||
def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType:
|
||||
|
||||
@@ -695,7 +695,6 @@ class EngineArgs:
|
||||
override_attention_dtype: str | None = ModelConfig.override_attention_dtype
|
||||
attention_backend: AttentionBackendEnum | None = AttentionConfig.backend
|
||||
|
||||
calculate_kv_scales: bool = CacheConfig.calculate_kv_scales
|
||||
kv_cache_dtype_skip_layers: list[str] = get_field(
|
||||
CacheConfig, "kv_cache_dtype_skip_layers"
|
||||
)
|
||||
@@ -1210,9 +1209,6 @@ class EngineArgs:
|
||||
cache_group.add_argument(
|
||||
"--prefix-caching-hash-algo", **cache_kwargs["prefix_caching_hash_algo"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--calculate-kv-scales", **cache_kwargs["calculate_kv_scales"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--kv-cache-dtype-skip-layers", **cache_kwargs["kv_cache_dtype_skip_layers"]
|
||||
)
|
||||
@@ -1957,7 +1953,6 @@ class EngineArgs:
|
||||
sliding_window=sliding_window,
|
||||
enable_prefix_caching=self.enable_prefix_caching,
|
||||
prefix_caching_hash_algo=self.prefix_caching_hash_algo,
|
||||
calculate_kv_scales=self.calculate_kv_scales,
|
||||
kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers,
|
||||
kv_sharing_fast_prefill=self.kv_sharing_fast_prefill,
|
||||
mamba_cache_dtype=self.mamba_cache_dtype,
|
||||
|
||||
@@ -1325,12 +1325,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB": lambda: maybe_convert_int(
|
||||
os.environ.get("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB", None)
|
||||
),
|
||||
# Divisor for dynamic query scale factor calculation for FP8 KV Cache
|
||||
"Q_SCALE_CONSTANT": lambda: int(os.getenv("Q_SCALE_CONSTANT", "200")),
|
||||
# Divisor for dynamic key scale factor calculation for FP8 KV Cache
|
||||
"K_SCALE_CONSTANT": lambda: int(os.getenv("K_SCALE_CONSTANT", "200")),
|
||||
# Divisor for dynamic value scale factor calculation for FP8 KV Cache
|
||||
"V_SCALE_CONSTANT": lambda: int(os.getenv("V_SCALE_CONSTANT", "100")),
|
||||
# If set, enable multiprocessing in LLM for the V1 code path.
|
||||
"VLLM_ENABLE_V1_MULTIPROCESSING": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1"))
|
||||
|
||||
@@ -144,11 +144,6 @@ def set_default_quant_scales(layer: nn.Module, register_buffer: bool = False) ->
|
||||
layer._v_scale_cpu = torch.tensor(1.0, dtype=torch.float32)
|
||||
layer._prob_scale_float = 1.0
|
||||
|
||||
# Initialize q/k/v range constants used by calc_kv_scales
|
||||
layer.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32)
|
||||
layer.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
|
||||
layer.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
|
||||
|
||||
|
||||
def _init_kv_cache_quant(
|
||||
layer: nn.Module,
|
||||
@@ -270,10 +265,8 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
vllm_config = get_current_vllm_config()
|
||||
if cache_config is not None:
|
||||
kv_cache_dtype = cache_config.cache_dtype
|
||||
calculate_kv_scales = cache_config.calculate_kv_scales
|
||||
else:
|
||||
kv_cache_dtype = "auto"
|
||||
calculate_kv_scales = False
|
||||
|
||||
# llm-compressor models declare an FP8 KV-cache scheme in their
|
||||
# checkpoint config. Honor it only when the user did not explicitly
|
||||
@@ -284,10 +277,8 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
kv_cache_scheme = getattr(quant_config, "kv_cache_scheme", None)
|
||||
if kv_cache_scheme is not None and kv_cache_dtype == "auto":
|
||||
kv_cache_dtype = "fp8"
|
||||
calculate_kv_scales = False
|
||||
if cache_config is not None:
|
||||
cache_config.cache_dtype = "fp8"
|
||||
cache_config.calculate_kv_scales = False
|
||||
|
||||
# Check if per-head quant scales are required based on kv_cache_scheme
|
||||
use_per_head_quant_scales = (
|
||||
@@ -312,7 +303,6 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
skip = True
|
||||
if skip:
|
||||
kv_cache_dtype = "auto"
|
||||
calculate_kv_scales = False
|
||||
logger.debug(
|
||||
"Layer %s: kv_cache_dtype=%s, sliding_window=%s",
|
||||
prefix,
|
||||
@@ -324,7 +314,6 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
kv_cache_dtype, vllm_config.model_config
|
||||
)
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
self.calculate_kv_scales = calculate_kv_scales
|
||||
if num_kv_heads is None:
|
||||
num_kv_heads = num_heads
|
||||
assert num_heads % num_kv_heads == 0, (
|
||||
@@ -505,10 +494,6 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
context using
|
||||
`vllm.forward_context.get_forward_context().attn_metadata`.
|
||||
"""
|
||||
if self.calculate_kv_scales:
|
||||
torch.ops.vllm.maybe_calc_kv_scales(
|
||||
query, key, value, _encode_layer_name(self.layer_name)
|
||||
)
|
||||
if output_dtype is None:
|
||||
output_dtype = query.dtype
|
||||
if self.query_quant is not None:
|
||||
@@ -581,18 +566,6 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
return output.view(-1, hidden_size)
|
||||
|
||||
def calc_kv_scales(self, query, key, value):
|
||||
self._q_scale.copy_(torch.abs(query).max() / self.q_range)
|
||||
self._k_scale.copy_(torch.abs(key).max() / self.k_range)
|
||||
self._v_scale.copy_(torch.abs(value).max() / self.v_range)
|
||||
self._q_scale_float = self._q_scale.item()
|
||||
self._k_scale_float = self._k_scale.item()
|
||||
self._v_scale_float = self._v_scale.item()
|
||||
self._k_scale_cpu.fill_(self._k_scale_float)
|
||||
self._v_scale_cpu.fill_(self._v_scale_float)
|
||||
# We only calculate the scales once
|
||||
self.calculate_kv_scales = False
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
s = f"head_size={self.impl.head_size}" # type: ignore
|
||||
s += f", num_heads={self.impl.num_heads}" # type: ignore
|
||||
@@ -694,41 +667,6 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
|
||||
|
||||
def maybe_calc_kv_scales(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
|
||||
# Only calculate if the layer's calculate_kv_scales flag is True
|
||||
# This flag gets set to False after the first forward pass
|
||||
if not self.calculate_kv_scales:
|
||||
return
|
||||
|
||||
self.calc_kv_scales(query, key, value)
|
||||
|
||||
|
||||
def maybe_calc_kv_scales_fake(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="maybe_calc_kv_scales",
|
||||
op_func=maybe_calc_kv_scales,
|
||||
mutates_args=["query", "key", "value"],
|
||||
fake_impl=maybe_calc_kv_scales_fake,
|
||||
)
|
||||
|
||||
|
||||
def get_attention_context(
|
||||
layer_name: str,
|
||||
) -> tuple[Any, "Attention | MLAAttention", torch.Tensor, torch.Tensor]:
|
||||
|
||||
@@ -399,10 +399,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
if cache_config is not None:
|
||||
kv_cache_dtype: CacheDType = cache_config.cache_dtype
|
||||
calculate_kv_scales = cache_config.calculate_kv_scales
|
||||
else:
|
||||
kv_cache_dtype = "auto"
|
||||
calculate_kv_scales = False
|
||||
self.quant_config = quant_config
|
||||
|
||||
if cache_config is not None and cache_config.kv_cache_dtype_skip_layers:
|
||||
@@ -411,7 +409,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
if str(layer_idx) in cache_config.kv_cache_dtype_skip_layers:
|
||||
kv_cache_dtype = "auto"
|
||||
calculate_kv_scales = False
|
||||
logger.debug(
|
||||
"Layer %s: kv_cache_dtype=%s",
|
||||
prefix,
|
||||
@@ -460,7 +457,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
# Initialize KV cache quantization attributes
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
self.calculate_kv_scales = calculate_kv_scales
|
||||
_init_kv_cache_quant(self, quant_config, prefix)
|
||||
|
||||
if (
|
||||
@@ -560,11 +556,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
and _vllm_config.parallel_config.dcp_comm_backend == "a2a"
|
||||
)
|
||||
|
||||
# Initialize q/k/v range constants.
|
||||
self.q_range = torch.tensor(envs.Q_SCALE_CONSTANT, dtype=torch.float32)
|
||||
self.k_range = torch.tensor(envs.K_SCALE_CONSTANT, dtype=torch.float32)
|
||||
self.v_range = torch.tensor(envs.V_SCALE_CONSTANT, dtype=torch.float32)
|
||||
|
||||
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
|
||||
|
||||
# If kv_b_proj_weight is unquantized, quantize it to mxfp4 if supported
|
||||
@@ -606,14 +597,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
output_shape: torch.Size | None = None,
|
||||
q_dcp_replicated: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.calculate_kv_scales:
|
||||
torch.ops.vllm.maybe_calc_kv_scales(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
_encode_layer_name(self.layer_name),
|
||||
)
|
||||
|
||||
if self.use_direct_call:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
@@ -1110,30 +1093,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
if not should_load_quant_weights(quant_method):
|
||||
set_default_quant_scales(self, register_buffer=False)
|
||||
|
||||
def calc_kv_scales(
|
||||
self, q: torch.Tensor, kv_c_normed: torch.Tensor, k_pe: torch.Tensor
|
||||
) -> None:
|
||||
"""Optional scale calculation for MLA inputs.
|
||||
|
||||
Mirrors Attention.calc_kv_scales. Not all MLA backends require this
|
||||
"""
|
||||
# Use safe defaults if ranges are not present
|
||||
q_range = getattr(self, "q_range", torch.tensor(1.0))
|
||||
k_range = getattr(self, "k_range", torch.tensor(1.0))
|
||||
v_range = getattr(self, "v_range", torch.tensor(1.0))
|
||||
|
||||
self._q_scale.copy_(torch.abs(q).max() / q_range)
|
||||
# kv_c_normed is the compressed KV representation; use it for k/v
|
||||
kv_abs_max = torch.abs(kv_c_normed).max()
|
||||
self._k_scale.copy_(kv_abs_max / k_range)
|
||||
self._v_scale.copy_(kv_abs_max / v_range)
|
||||
self._q_scale_float = self._q_scale.item()
|
||||
self._k_scale_float = self._k_scale.item()
|
||||
self._v_scale_float = self._v_scale.item()
|
||||
self._k_scale_cpu.fill_(self._k_scale_float)
|
||||
self._v_scale_cpu.fill_(self._v_scale_float)
|
||||
self.calculate_kv_scales = False
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return self.attn_backend
|
||||
|
||||
|
||||
@@ -80,8 +80,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
return
|
||||
|
||||
# Per-token-head quantized KV cache: scales are computed dynamically
|
||||
# per (token, head) in the kernel at cache-write time. Checkpoint
|
||||
# scales are never used regardless of calculate_kv_scales.
|
||||
# per (token, head) in the kernel at cache-write time.
|
||||
if kv_cache_uses_per_token_head_scales(layer.kv_cache_dtype):
|
||||
layer._k_scale.copy_(1.0)
|
||||
layer._v_scale.copy_(1.0)
|
||||
@@ -97,10 +96,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
# regardless whether the kv-scale is available in the checkpoint.
|
||||
# No need to process kv scales after loading if we are going to
|
||||
# calculate them on the fly.
|
||||
if (
|
||||
is_quantized_kv_cache(layer.kv_cache_dtype)
|
||||
and not layer.calculate_kv_scales
|
||||
):
|
||||
if is_quantized_kv_cache(layer.kv_cache_dtype):
|
||||
if layer.k_scale > 0.0 and layer.v_scale > 0.0:
|
||||
# We prefer to use separate k_scale and v_scale if present
|
||||
k_scale = layer.k_scale.to("cpu").tolist()
|
||||
@@ -158,7 +154,6 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
q_scale = layer.q_scale
|
||||
if current_platform.is_fp8_fnuz():
|
||||
q_scale *= 2
|
||||
layer.calculate_kv_scales = False
|
||||
else:
|
||||
q_scale = 1.0
|
||||
if layer.prob_scale > 0.0:
|
||||
|
||||
@@ -445,23 +445,6 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
|
||||
Args:
|
||||
vllm_config: vLLM Config
|
||||
"""
|
||||
cache_config = vllm_config.cache_config
|
||||
|
||||
# Disable calculate_kv_scales for hybrid models: uninitialized
|
||||
# recurrent state corrupts scales during the calibration pass.
|
||||
# See issue: https://github.com/vllm-project/vllm/issues/37554
|
||||
|
||||
if cache_config.calculate_kv_scales:
|
||||
logger.warning(
|
||||
"Disabling calculate_kv_scales for hybrid model '%s'. "
|
||||
"Hybrid models with recurrent layers (GDN, Mamba, SSM) "
|
||||
"produce unreliable KV cache scales during the "
|
||||
"calibration pass because recurrent state is "
|
||||
"uninitialized. Using default scale of 1.0 instead.",
|
||||
vllm_config.model_config.model,
|
||||
)
|
||||
cache_config.calculate_kv_scales = False
|
||||
|
||||
# Enable FULL_AND_PIECEWISE by default
|
||||
MambaModelConfig.verify_and_update_config(vllm_config)
|
||||
|
||||
|
||||
@@ -642,7 +642,6 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
# MiniMax-M3 sparse attention owns its KV-cache insert/read path instead
|
||||
# of wrapping the generic Attention module. Keep the same runtime scale
|
||||
# attributes so FP8 KV reads can honor vLLM's per-layer descale contract.
|
||||
self.calculate_kv_scales = False
|
||||
set_default_quant_scales(self, register_buffer=True)
|
||||
|
||||
# Shared top-k buffer: the indexer writes the selected blocks into it and
|
||||
|
||||
@@ -495,7 +495,6 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
# MiniMax-M3 sparse attention owns its KV-cache insert/read path instead
|
||||
# of wrapping the generic Attention module. Keep the same runtime scale
|
||||
# attributes so FP8 KV reads can honor vLLM's per-layer descale contract.
|
||||
self.calculate_kv_scales = False
|
||||
set_default_quant_scales(self, register_buffer=True)
|
||||
# Indexer side-cache dtype, mirroring --kv-cache-dtype for the main
|
||||
# cache (--attention-config '{"indexer_kv_dtype": ...}').
|
||||
|
||||
@@ -499,7 +499,6 @@ class GPUModelRunner(
|
||||
self.max_model_len = model_config.max_model_len
|
||||
|
||||
# Always set to false after the first forward pass
|
||||
self.calculate_kv_scales = self.cache_config.calculate_kv_scales
|
||||
self.dcp_world_size = self.parallel_config.decode_context_parallel_size
|
||||
self.dcp_rank = 0 if self.dcp_world_size <= 1 else get_dcp_group().rank_in_group
|
||||
self.max_num_tokens = scheduler_config.max_num_batched_tokens
|
||||
@@ -4402,14 +4401,6 @@ class GPUModelRunner(
|
||||
scheduler_output, num_tokens_padded, intermediate_tensors
|
||||
)
|
||||
|
||||
# Set cudagraph mode to none if calc_kv_scales is true.
|
||||
# KV scales calculation involves dynamic operations that are incompatible
|
||||
# with CUDA graph capture.
|
||||
if self.calculate_kv_scales:
|
||||
cudagraph_mode = CUDAGraphMode.NONE
|
||||
# Mark KV scales as calculated after the first forward pass
|
||||
self.calculate_kv_scales = False
|
||||
|
||||
# Encoder-decoder models can only compile the pure decode steps where no
|
||||
# encoder inputs are present. Use eager for the first pass.
|
||||
num_encoder_reqs = len(scheduler_output.scheduled_encoder_inputs)
|
||||
|
||||
Reference in New Issue
Block a user