From 4ff028d77e063e0055849ffec3855c6634179158 Mon Sep 17 00:00:00 2001 From: Stefano Castagnetta Date: Mon, 5 Oct 2026 14:13:30 +0200 Subject: [PATCH] [Bugfix][Qwen4Exp] Honor --kv-cache-dtype-skip-layers in QSA attention (#60023) Signed-off-by: Stefano Castagnetta Co-authored-by: Thien Tran --- tests/models/qwen4_exp/test_qsa_reference.py | 14 ++++++++++++++ vllm/models/qwen4_exp/nvidia/qsa.py | 17 ++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/models/qwen4_exp/test_qsa_reference.py b/tests/models/qwen4_exp/test_qsa_reference.py index dc431aed7a..bdcc4e3423 100644 --- a/tests/models/qwen4_exp/test_qsa_reference.py +++ b/tests/models/qwen4_exp/test_qsa_reference.py @@ -15,6 +15,7 @@ from vllm.models.qwen4_exp.nvidia import ( ) from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops from vllm.models.qwen4_exp.nvidia.ops import qsa_indexer as qsa_indexer_ops +from vllm.models.qwen4_exp.nvidia.qsa import qsa_kv_cache_dtype from vllm.platforms import current_platform from vllm.triton_utils import HAS_TRITON from vllm.v1.worker.utils import clear_layer_kv_caches @@ -448,6 +449,19 @@ def test_qsa_ring_capacity_covers_one_speculative_step( assert spec.block_size == expected +def test_qsa_kv_cache_dtype_honors_skip_layers() -> None: + """``--kv-cache-dtype-skip-layers`` keeps the listed QSA layers unquantized. + + The MTP layer's own attention is the one that matters on Flash-Next: FP8 + there cuts draft acceptance at depth while the target layers stay FP8. + """ + cache_config = SimpleNamespace(cache_dtype="fp8", kv_cache_dtype_skip_layers=["48"]) + assert qsa_kv_cache_dtype(cache_config, "mtp.layers.48.self_attn") == "auto" + assert qsa_kv_cache_dtype(cache_config, "model.layers.47.self_attn") == "fp8" + cache_config.kv_cache_dtype_skip_layers = [] + assert qsa_kv_cache_dtype(cache_config, "mtp.layers.48.self_attn") == "fp8" + + @requires_qsa_kernels def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None: device = torch.device("cuda") diff --git a/vllm/models/qwen4_exp/nvidia/qsa.py b/vllm/models/qwen4_exp/nvidia/qsa.py index 6877189768..a20fcdeec9 100644 --- a/vllm/models/qwen4_exp/nvidia/qsa.py +++ b/vllm/models/qwen4_exp/nvidia/qsa.py @@ -13,7 +13,7 @@ from transformers import Qwen4ExpTextConfig from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import VllmConfig -from vllm.config.cache import CacheDType +from vllm.config.cache import CacheConfig, CacheDType from vllm.distributed import get_tensor_model_parallel_world_size from vllm.forward_context import get_forward_context from vllm.model_executor.layers.attention.attention import ( @@ -29,7 +29,11 @@ from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope from vllm.model_executor.models.qwen3_next import Qwen3NextAttention -from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper +from vllm.model_executor.models.utils import ( + AutoWeightsLoader, + WeightsMapper, + extract_layer_index, +) from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability from vllm.utils.torch_utils import ( @@ -125,6 +129,13 @@ class Qwen4ExpQSAQKVIndexerLinear(MergedColumnParallelLinear): return super().load_weights(remap_shards()) +def qsa_kv_cache_dtype(cache_config: CacheConfig, prefix: str) -> CacheDType: + """The layer's KV cache dtype, honoring ``--kv-cache-dtype-skip-layers``.""" + if str(extract_layer_index(prefix)) in cache_config.kv_cache_dtype_skip_layers: + return "auto" + return cache_config.cache_dtype + + class Qwen4ExpQSAMetadataBuilder(FlashAttentionMetadataBuilder): """Flash metadata supporting uniform decode and target-verify graphs.""" @@ -444,7 +455,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): self.layer_name = f"{prefix}.attn" self.attn_type = AttentionType.DECODER - self.kv_cache_dtype = cache_config.cache_dtype + self.kv_cache_dtype = qsa_kv_cache_dtype(cache_config, prefix) self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype( self.kv_cache_dtype, model_config )