[Bugfix][Qwen4Exp] Honor --kv-cache-dtype-skip-layers in QSA attention (#60023)

Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Stefano Castagnetta
2026-10-05 12:13:30 +00:00
committed by GitHub
co-authored by Thien Tran
parent 55b80221ed
commit 4ff028d77e
2 changed files with 28 additions and 3 deletions
@@ -15,6 +15,7 @@ from vllm.models.qwen4_exp.nvidia import (
)
from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops
from vllm.models.qwen4_exp.nvidia.ops import qsa_indexer as qsa_indexer_ops
from vllm.models.qwen4_exp.nvidia.qsa import qsa_kv_cache_dtype
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
from vllm.v1.worker.utils import clear_layer_kv_caches
@@ -448,6 +449,19 @@ def test_qsa_ring_capacity_covers_one_speculative_step(
assert spec.block_size == expected
def test_qsa_kv_cache_dtype_honors_skip_layers() -> None:
"""``--kv-cache-dtype-skip-layers`` keeps the listed QSA layers unquantized.
The MTP layer's own attention is the one that matters on Flash-Next: FP8
there cuts draft acceptance at depth while the target layers stay FP8.
"""
cache_config = SimpleNamespace(cache_dtype="fp8", kv_cache_dtype_skip_layers=["48"])
assert qsa_kv_cache_dtype(cache_config, "mtp.layers.48.self_attn") == "auto"
assert qsa_kv_cache_dtype(cache_config, "model.layers.47.self_attn") == "fp8"
cache_config.kv_cache_dtype_skip_layers = []
assert qsa_kv_cache_dtype(cache_config, "mtp.layers.48.self_attn") == "fp8"
@requires_qsa_kernels
def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
device = torch.device("cuda")