[Bugfix][Qwen4Exp] Honor --kv-cache-dtype-skip-layers in QSA attention (#60023)
Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
co-authored by
Thien Tran
parent
55b80221ed
commit
4ff028d77e
@@ -15,6 +15,7 @@ from vllm.models.qwen4_exp.nvidia import (
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ops import qsa as qsa_ops
|
||||
from vllm.models.qwen4_exp.nvidia.ops import qsa_indexer as qsa_indexer_ops
|
||||
from vllm.models.qwen4_exp.nvidia.qsa import qsa_kv_cache_dtype
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
from vllm.v1.worker.utils import clear_layer_kv_caches
|
||||
@@ -448,6 +449,19 @@ def test_qsa_ring_capacity_covers_one_speculative_step(
|
||||
assert spec.block_size == expected
|
||||
|
||||
|
||||
def test_qsa_kv_cache_dtype_honors_skip_layers() -> None:
|
||||
"""``--kv-cache-dtype-skip-layers`` keeps the listed QSA layers unquantized.
|
||||
|
||||
The MTP layer's own attention is the one that matters on Flash-Next: FP8
|
||||
there cuts draft acceptance at depth while the target layers stay FP8.
|
||||
"""
|
||||
cache_config = SimpleNamespace(cache_dtype="fp8", kv_cache_dtype_skip_layers=["48"])
|
||||
assert qsa_kv_cache_dtype(cache_config, "mtp.layers.48.self_attn") == "auto"
|
||||
assert qsa_kv_cache_dtype(cache_config, "model.layers.47.self_attn") == "fp8"
|
||||
cache_config.kv_cache_dtype_skip_layers = []
|
||||
assert qsa_kv_cache_dtype(cache_config, "mtp.layers.48.self_attn") == "fp8"
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
|
||||
device = torch.device("cuda")
|
||||
|
||||
@@ -13,7 +13,7 @@ from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.config.cache import CacheConfig, CacheDType
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention.attention import (
|
||||
@@ -29,7 +29,11 @@ from vllm.model_executor.layers.linear import (
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope
|
||||
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.torch_utils import (
|
||||
@@ -125,6 +129,13 @@ class Qwen4ExpQSAQKVIndexerLinear(MergedColumnParallelLinear):
|
||||
return super().load_weights(remap_shards())
|
||||
|
||||
|
||||
def qsa_kv_cache_dtype(cache_config: CacheConfig, prefix: str) -> CacheDType:
|
||||
"""The layer's KV cache dtype, honoring ``--kv-cache-dtype-skip-layers``."""
|
||||
if str(extract_layer_index(prefix)) in cache_config.kv_cache_dtype_skip_layers:
|
||||
return "auto"
|
||||
return cache_config.cache_dtype
|
||||
|
||||
|
||||
class Qwen4ExpQSAMetadataBuilder(FlashAttentionMetadataBuilder):
|
||||
"""Flash metadata supporting uniform decode and target-verify graphs."""
|
||||
|
||||
@@ -444,7 +455,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
|
||||
self.layer_name = f"{prefix}.attn"
|
||||
self.attn_type = AttentionType.DECODER
|
||||
self.kv_cache_dtype = cache_config.cache_dtype
|
||||
self.kv_cache_dtype = qsa_kv_cache_dtype(cache_config, prefix)
|
||||
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
|
||||
self.kv_cache_dtype, model_config
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user