Signed-off-by: kimseunghyun <hellonypcteam@gmail.com> Signed-off-by: kimseunghyun <incongnito12@gmail.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: kimseunghyun <hellonypcteam@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
673 lines
25 KiB
Python
673 lines
25 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""NVIDIA QSA owner with Triton kernels."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Callable, Iterable
|
|
from typing import ClassVar, cast
|
|
|
|
import torch
|
|
from torch import nn
|
|
from transformers import Qwen4ExpTextConfig
|
|
|
|
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
|
from vllm.config import VllmConfig
|
|
from vllm.config.cache import CacheConfig, CacheDType
|
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
|
from vllm.forward_context import get_forward_context
|
|
from vllm.model_executor.layers.attention.attention import (
|
|
set_default_quant_scales,
|
|
)
|
|
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
|
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
|
from vllm.model_executor.layers.linear import (
|
|
MergedColumnParallelLinear,
|
|
QKVParallelLinear,
|
|
RowParallelLinear,
|
|
)
|
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
|
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope
|
|
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
|
from vllm.model_executor.models.qwen3_vl import mrope_positions_factor
|
|
from vllm.model_executor.models.utils import (
|
|
AutoWeightsLoader,
|
|
WeightsMapper,
|
|
extract_layer_index,
|
|
)
|
|
from vllm.platforms import current_platform
|
|
from vllm.platforms.interface import DeviceCapability
|
|
from vllm.utils.torch_utils import (
|
|
kv_cache_dtype_str_to_dtype,
|
|
)
|
|
from vllm.v1.attention.backend import (
|
|
AttentionBackend,
|
|
AttentionCGSupport,
|
|
AttentionType,
|
|
MultipleOf,
|
|
)
|
|
from vllm.v1.attention.backends.fa_utils import is_flash_attn_varlen_func_available
|
|
from vllm.v1.attention.backends.flash_attn import (
|
|
FlashAttentionBackend,
|
|
FlashAttentionImpl,
|
|
FlashAttentionMetadata,
|
|
FlashAttentionMetadataBuilder,
|
|
)
|
|
from vllm.v1.kv_cache_interface import (
|
|
FullAttentionSpec,
|
|
KVCacheSpec,
|
|
get_kv_quant_mode,
|
|
)
|
|
|
|
from ..common.qsa_cache import QSAForwardMetadata
|
|
from . import model
|
|
from .indexer_qsa import QSAIndexer
|
|
|
|
|
|
class Qwen4ExpQSAQKVIndexerLinear(MergedColumnParallelLinear):
|
|
"""Pack sharded Q/gate, K, V and replicated indexer Q/K in one GEMM."""
|
|
|
|
def __init__(
|
|
self,
|
|
qkv_proj: QKVParallelLinear,
|
|
index_size: int,
|
|
quant_config: QuantizationConfig | None,
|
|
) -> None:
|
|
self.num_kv_head_replicas = qkv_proj.num_kv_head_replicas
|
|
super().__init__(
|
|
input_size=qkv_proj.input_size,
|
|
output_sizes=[*qkv_proj.output_sizes, index_size * qkv_proj.tp_size],
|
|
bias=False,
|
|
params_dtype=qkv_proj.params_dtype,
|
|
quant_config=quant_config,
|
|
prefix=qkv_proj.prefix,
|
|
)
|
|
|
|
def _load_shard(
|
|
self,
|
|
loader: Callable[..., None],
|
|
param: nn.Parameter,
|
|
loaded_weight: torch.Tensor,
|
|
loaded_shard_id: int | tuple[int, ...] | None,
|
|
) -> None:
|
|
tp_rank = self.tp_rank
|
|
param_tp_rank = getattr(param, "tp_rank", None)
|
|
if loaded_shard_id == 3:
|
|
shard_rank = 0
|
|
elif loaded_shard_id in (1, 2):
|
|
shard_rank = tp_rank // self.num_kv_head_replicas
|
|
else:
|
|
shard_rank = tp_rank
|
|
self.tp_rank = shard_rank
|
|
if param_tp_rank is not None:
|
|
param.tp_rank = shard_rank
|
|
try:
|
|
loader(param, loaded_weight, loaded_shard_id)
|
|
finally:
|
|
self.tp_rank = tp_rank
|
|
if param_tp_rank is not None:
|
|
param.tp_rank = param_tp_rank
|
|
|
|
def weight_loader(self, param, loaded_weight, loaded_shard_id=None) -> None:
|
|
self._load_shard(super().weight_loader, param, loaded_weight, loaded_shard_id)
|
|
|
|
def weight_loader_v2(self, param, loaded_weight, loaded_shard_id=None) -> None:
|
|
self._load_shard(
|
|
super().weight_loader_v2, param, loaded_weight, loaded_shard_id
|
|
)
|
|
|
|
def load_weights(
|
|
self, weights: Iterable[tuple[str, torch.Tensor]]
|
|
) -> Iterable[str]:
|
|
def remap_shards():
|
|
for name, weight in weights:
|
|
shard_id = getattr(weight, "shard_id", None)
|
|
if isinstance(shard_id, str):
|
|
weight = weight.detach()
|
|
weight.shard_id = {"q": 0, "k": 1, "v": 2}[shard_id]
|
|
yield name, weight
|
|
|
|
return super().load_weights(remap_shards())
|
|
|
|
|
|
def qsa_kv_cache_dtype(cache_config: CacheConfig, prefix: str) -> CacheDType:
|
|
"""The layer's KV cache dtype, honoring ``--kv-cache-dtype-skip-layers``."""
|
|
if str(extract_layer_index(prefix)) in cache_config.kv_cache_dtype_skip_layers:
|
|
return "auto"
|
|
return cache_config.cache_dtype
|
|
|
|
|
|
class Qwen4ExpQSAMetadataBuilder(FlashAttentionMetadataBuilder):
|
|
"""Flash metadata supporting uniform decode and target-verify graphs."""
|
|
|
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
|
|
|
|
|
class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
|
|
"""FullAttentionSpec backend used by the merged QSA owner."""
|
|
|
|
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
|
# fp8/fp8_e4m3: e4m3 bytes in a uint8 cache, written by reshape_and_cache
|
|
# with the layer's per-tensor scales and dequantized on load inside the QSA
|
|
# Triton kernel. flash-attn never runs over this cache, so its fp8 probe
|
|
# does not apply (see supports_kv_cache_dtype and the impl constructor).
|
|
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
|
"auto",
|
|
"bfloat16",
|
|
"fp8",
|
|
"fp8_e4m3",
|
|
]
|
|
|
|
@classmethod
|
|
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
|
|
return kv_cache_dtype is None or kv_cache_dtype in cls.supported_kv_cache_dtypes
|
|
|
|
@classmethod
|
|
def supports_combination(
|
|
cls,
|
|
head_size: int,
|
|
dtype: torch.dtype,
|
|
kv_cache_dtype: CacheDType | None,
|
|
block_size: int | None,
|
|
use_mla: bool,
|
|
has_sink: bool,
|
|
use_sparse: bool,
|
|
use_mm_prefix: bool,
|
|
device_capability: DeviceCapability,
|
|
) -> str | None:
|
|
# QSA dequantizes the fp8 KV in its own Triton kernel and never runs
|
|
# flash-attn over the quantized cache, so the parent's fp8-KV rejection
|
|
# does not apply and every combination it is handed is accepted here.
|
|
return None
|
|
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "QWEN4_EXP_QSA_TRITON"
|
|
|
|
@staticmethod
|
|
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
|
# QSA consumes manager pages directly and does not use FA4 paged attention.
|
|
return [MultipleOf(16)]
|
|
|
|
@staticmethod
|
|
def get_impl_cls() -> type[Qwen4ExpQSAFlashAttentionImpl]:
|
|
return Qwen4ExpQSAFlashAttentionImpl
|
|
|
|
@staticmethod
|
|
def get_builder_cls() -> type[Qwen4ExpQSAMetadataBuilder]:
|
|
return Qwen4ExpQSAMetadataBuilder
|
|
|
|
@classmethod
|
|
def is_sparse(cls) -> bool:
|
|
return True
|
|
|
|
@classmethod
|
|
def supports_kv_connector(cls) -> bool:
|
|
return False
|
|
|
|
|
|
class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
|
"""Run paged sparse GQA with the QSA Triton kernel."""
|
|
|
|
supports_dcp: bool = False
|
|
supports_pcp: bool = False
|
|
|
|
def __init__(
|
|
self,
|
|
num_heads: int,
|
|
head_size: int,
|
|
scale: float,
|
|
num_kv_heads: int,
|
|
alibi_slopes: list[float] | None,
|
|
sliding_window: int | None,
|
|
kv_cache_dtype: str,
|
|
logits_soft_cap: float | None = None,
|
|
attn_type: AttentionType = AttentionType.DECODER,
|
|
kv_sharing_target_layer_name: str | None = None,
|
|
sinks: torch.Tensor | None = None,
|
|
) -> None:
|
|
# The parent constructor probes flash-attn for quantized-KV support and
|
|
# raises where it is unavailable (sm120), but QSA dequantizes fp8 inside
|
|
# its own Triton kernel and never runs flash-attn over the cache. Hand
|
|
# the parent "auto" for that probe and restore the real dtype afterwards:
|
|
# the parent only uses it there, and do_kv_cache_update reads the
|
|
# attribute at call time.
|
|
real_kv_cache_dtype = kv_cache_dtype
|
|
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
|
kv_cache_dtype = "auto"
|
|
super().__init__(
|
|
num_heads,
|
|
head_size,
|
|
scale,
|
|
num_kv_heads,
|
|
alibi_slopes,
|
|
sliding_window,
|
|
kv_cache_dtype,
|
|
logits_soft_cap,
|
|
attn_type,
|
|
kv_sharing_target_layer_name,
|
|
sinks,
|
|
)
|
|
self.kv_cache_dtype = real_kv_cache_dtype
|
|
if not is_flash_attn_varlen_func_available():
|
|
raise NotImplementedError("Qwen4Exp QSA requires FlashAttention")
|
|
if self.dcp_world_size != 1:
|
|
raise NotImplementedError(
|
|
"Qwen4Exp QSA does not support decode context parallelism"
|
|
)
|
|
if self.kv_cache_dtype not in ("auto", "bfloat16", "fp8", "fp8_e4m3"):
|
|
raise NotImplementedError(
|
|
"Qwen4Exp QSA requires a BF16 or FP8-e4m3 main KV cache"
|
|
)
|
|
self.supports_quant_query_input = False
|
|
|
|
def forward_qsa(
|
|
self,
|
|
layer: torch.nn.Module,
|
|
query: torch.Tensor,
|
|
key: torch.Tensor,
|
|
value: torch.Tensor,
|
|
kv_cache: torch.Tensor,
|
|
attn_metadata: FlashAttentionMetadata,
|
|
output: torch.Tensor,
|
|
token_to_req: torch.Tensor,
|
|
use_prefill_config: bool,
|
|
output_gate: torch.Tensor,
|
|
output_scale: torch.Tensor | None = None,
|
|
output_block_scale: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
del key, value
|
|
if output_scale is not None or output_block_scale is not None:
|
|
raise NotImplementedError("QSA does not support fused output quantization")
|
|
if self.alibi_slopes is not None or self.sinks is not None:
|
|
raise NotImplementedError("QSA does not support ALiBi or attention sinks")
|
|
if self.sliding_window != (-1, -1):
|
|
raise NotImplementedError("QSA does not support sliding-window attention")
|
|
|
|
num_tokens = attn_metadata.num_actual_tokens
|
|
output.zero_()
|
|
if num_tokens == 0:
|
|
return output
|
|
|
|
topk_buffer = getattr(layer, "topk_indices_buffer", None)
|
|
if topk_buffer is None:
|
|
raise RuntimeError("QSA owner did not provide its top-k buffer")
|
|
logical_indices = topk_buffer[:num_tokens]
|
|
token_to_req = token_to_req[:num_tokens]
|
|
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
|
k_scale = v_scale = None
|
|
if self.kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
|
# The cache is allocated as uint8; on sm_89+ reinterpret the e4m3
|
|
# bytes (same itemsize, so shape and strides are preserved). Below
|
|
# sm_89 the kernel byte-decodes the raw uint8 storage instead.
|
|
if current_platform.has_device_capability(89):
|
|
key_cache = key_cache.view(torch.float8_e4m3fn)
|
|
value_cache = value_cache.view(torch.float8_e4m3fn)
|
|
# Host-side per-tensor dequant scales (Python floats), as used by
|
|
# other host-scale backends; folded into the kernel's scales.
|
|
k_scale = layer._k_scale_float
|
|
v_scale = layer._v_scale_float
|
|
if query.dtype != torch.bfloat16 or key_cache.dtype not in (
|
|
torch.bfloat16,
|
|
torch.float8_e4m3fn,
|
|
torch.uint8,
|
|
):
|
|
raise NotImplementedError(
|
|
"Qwen4Exp QSA requires BF16 Q and BF16 or FP8-e4m3 K/V"
|
|
)
|
|
|
|
from .ops.qsa import qsa_sparse_paged_attention
|
|
|
|
qsa_sparse_paged_attention(
|
|
query[:num_tokens],
|
|
key_cache,
|
|
value_cache,
|
|
logical_indices,
|
|
attn_metadata.block_table,
|
|
token_to_req,
|
|
use_prefill_config,
|
|
output[:num_tokens],
|
|
k_scale=k_scale,
|
|
v_scale=v_scale,
|
|
output_gate=output_gate[:num_tokens],
|
|
)
|
|
return output
|
|
|
|
|
|
class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
|
"""Merged Qwen full-attention owner with a QSA index side branch."""
|
|
|
|
supports_dcp = False
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
vllm_config: VllmConfig,
|
|
config: Qwen4ExpTextConfig,
|
|
layer_id: int,
|
|
quant_config: QuantizationConfig | None = None,
|
|
reduce_results: bool = True,
|
|
prefix: str = "",
|
|
) -> None:
|
|
nn.Module.__init__(self)
|
|
cache_config = vllm_config.cache_config
|
|
model_config = vllm_config.model_config
|
|
if cache_config is None:
|
|
raise ValueError("Qwen4Exp QSA requires a paged KV cache")
|
|
if model_config.dtype != torch.bfloat16:
|
|
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
|
|
if cache_config.cache_dtype not in (
|
|
"auto",
|
|
"bfloat16",
|
|
"fp8",
|
|
"fp8_e4m3",
|
|
):
|
|
raise NotImplementedError(
|
|
"Qwen4Exp QSA requires a BF16 or FP8-e4m3 main KV cache"
|
|
)
|
|
if getattr(quant_config, "kv_cache_scheme", None) is not None:
|
|
raise NotImplementedError("Qwen4Exp QSA does not support KV quantization")
|
|
parallel_config = vllm_config.parallel_config
|
|
if (
|
|
parallel_config.prefill_context_parallel_size > 1
|
|
or parallel_config.decode_context_parallel_size > 1
|
|
):
|
|
raise NotImplementedError(
|
|
"Qwen4Exp QSA does not support context parallelism"
|
|
)
|
|
if not getattr(config, "is_causal", True):
|
|
raise NotImplementedError("Qwen4Exp QSA requires causal decoder attention")
|
|
|
|
self.config = config
|
|
self.hidden_size = int(config.hidden_size)
|
|
tp_size = get_tensor_model_parallel_world_size()
|
|
self.total_num_heads = int(config.num_attention_heads)
|
|
if self.total_num_heads % tp_size:
|
|
raise ValueError("QSA attention heads must be divisible by TP size")
|
|
self.num_heads = self.total_num_heads // tp_size
|
|
# Decode/verify batches have at most 1 + num_spec query tokens per
|
|
# request; use_prefill_config (max_query_len > this) steers the
|
|
# config table. Shorter batches take the decode profile — harmless,
|
|
# the difference is tile-shape tuning, not correctness.
|
|
self._max_decode_query_len = 1 + vllm_config.num_speculative_tokens
|
|
self.total_num_kv_heads = int(config.num_key_value_heads)
|
|
if self.total_num_kv_heads >= tp_size:
|
|
if self.total_num_kv_heads % tp_size:
|
|
raise ValueError("QSA KV heads must be divisible by TP size")
|
|
elif tp_size % self.total_num_kv_heads:
|
|
raise ValueError("TP size must be divisible by replicated QSA KV heads")
|
|
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
|
self.head_dim = int(config.head_dim or self.hidden_size // self.total_num_heads)
|
|
self.q_size = self.num_heads * self.head_dim
|
|
self.kv_size = self.num_kv_heads * self.head_dim
|
|
self.scaling = self.head_dim**-0.5
|
|
self.dual_chunk_attention_config = getattr(
|
|
config, "dual_chunk_attention_config", None
|
|
)
|
|
if self.dual_chunk_attention_config is not None:
|
|
raise NotImplementedError("Qwen4Exp QSA does not support dual-chunk RoPE")
|
|
# Qwen4Exp full-attention checkpoints always pack a sigmoid output
|
|
# gate next to Q, even when an inherited config default says otherwise.
|
|
self.attn_output_gate = True
|
|
|
|
self.qkv_proj = QKVParallelLinear(
|
|
self.hidden_size,
|
|
self.head_dim,
|
|
self.total_num_heads * (1 + self.attn_output_gate),
|
|
self.total_num_kv_heads,
|
|
bias=False,
|
|
quant_config=model.without_modelopt_fp4(quant_config),
|
|
prefix=f"{prefix}.qkv_proj",
|
|
)
|
|
self.o_proj = RowParallelLinear(
|
|
self.total_num_heads * self.head_dim,
|
|
self.hidden_size,
|
|
bias=False,
|
|
reduce_results=reduce_results,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.o_proj",
|
|
)
|
|
self.rotary_emb = get_rope(
|
|
head_size=self.head_dim,
|
|
max_position=config.max_position_embeddings,
|
|
rope_parameters=config.rope_parameters,
|
|
mrope_positions_factor=mrope_positions_factor(vllm_config),
|
|
)
|
|
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
|
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
|
|
|
mm_config = model_config.multimodal_config
|
|
text_only = mm_config is None or mm_config.language_model_only
|
|
mrope_section = getattr(self.rotary_emb, "mrope_section", None)
|
|
supports_mrope = bool(
|
|
type(self.rotary_emb) is MRotaryEmbedding
|
|
and mrope_section
|
|
and len(mrope_section) == 3
|
|
and sum(mrope_section) == self.rotary_emb.rotary_dim // 2
|
|
and getattr(self.rotary_emb, "mrope_interleaved", False)
|
|
)
|
|
supports_dtype = getattr(self.rotary_emb, "dtype", None) in (
|
|
torch.float16,
|
|
torch.bfloat16,
|
|
)
|
|
self.use_fused_qk_norm_rope_gate = (
|
|
self.attn_output_gate
|
|
and getattr(self.rotary_emb, "is_neox_style", False)
|
|
and current_platform.is_cuda()
|
|
and supports_dtype
|
|
and (text_only or supports_mrope)
|
|
)
|
|
|
|
self.layer_name = f"{prefix}.attn"
|
|
self.attn_type = AttentionType.DECODER
|
|
self.kv_cache_dtype = qsa_kv_cache_dtype(cache_config, prefix)
|
|
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
|
|
self.kv_cache_dtype, model_config
|
|
)
|
|
if self.kv_cache_torch_dtype not in (torch.bfloat16, torch.uint8):
|
|
raise NotImplementedError(
|
|
"Qwen4Exp QSA requires BF16 or FP8-e4m3 (uint8) cache storage"
|
|
)
|
|
self.kv_sharing_target_layer_name = None
|
|
self.kv_cache = torch.tensor([])
|
|
set_default_quant_scales(self, register_buffer=True)
|
|
|
|
self.attn_backend = Qwen4ExpQSAFlashAttentionBackend
|
|
self.impl = Qwen4ExpQSAFlashAttentionImpl(
|
|
self.num_heads,
|
|
self.head_dim,
|
|
self.scaling,
|
|
self.num_kv_heads,
|
|
None,
|
|
None,
|
|
self.kv_cache_dtype,
|
|
None,
|
|
AttentionType.DECODER,
|
|
None,
|
|
)
|
|
self.indexer = QSAIndexer(
|
|
vllm_config=vllm_config,
|
|
config=config,
|
|
layer_id=layer_id,
|
|
rotary_emb=self.rotary_emb,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.indexer",
|
|
)
|
|
# One launch does the indexer prepare, the main QK-norm/RoPE/gate and
|
|
# the main K/V cache write (see QSAIndexer.forward); otherwise all of
|
|
# them take the separate kernels. The fused writer receives an fp8
|
|
# pointer, which Triton cannot lower below sm_89.
|
|
self.use_fused_qsa_prepare = (
|
|
self.use_fused_qk_norm_rope_gate
|
|
and self.indexer.use_fused_pre_indexer
|
|
and (
|
|
self.kv_cache_torch_dtype != torch.uint8
|
|
or current_platform.has_device_capability(89)
|
|
)
|
|
)
|
|
self.fuse_indexer_projection = vllm_config.lora_config is None
|
|
if self.fuse_indexer_projection:
|
|
self.index_qk_size = self.indexer.index_qk_proj.output_size
|
|
self.qkv_proj = Qwen4ExpQSAQKVIndexerLinear(
|
|
self.qkv_proj,
|
|
self.index_qk_size,
|
|
model.without_modelopt_fp4(quant_config),
|
|
)
|
|
del self.indexer.index_qk_proj
|
|
|
|
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
|
# PACKED selection buffer: the trailing column holds each row's
|
|
# valid-entry count (written by the expand kernel) — never a token
|
|
# index; the sparse attention kernel reads it as its loop bound.
|
|
# MTP skip_topk steps reuse rows frozen from step 0; the count is
|
|
# a row column, so compaction/reuse keep it paired with the content.
|
|
self.register_buffer(
|
|
"topk_indices_buffer",
|
|
torch.empty(
|
|
max_tokens,
|
|
self.indexer.packed_output_width,
|
|
dtype=torch.int32,
|
|
),
|
|
persistent=False,
|
|
)
|
|
|
|
static_context = vllm_config.compilation_config.static_forward_context
|
|
if self.layer_name in static_context:
|
|
raise ValueError(f"Duplicate layer name: {self.layer_name}")
|
|
static_context[self.layer_name] = self
|
|
|
|
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
|
mapper = None
|
|
if self.fuse_indexer_projection:
|
|
mapper = WeightsMapper(
|
|
orig_to_new_stacked={
|
|
"indexer.index_qk_proj.": ("qkv_proj.", 3),
|
|
}
|
|
)
|
|
return AutoWeightsLoader(self).load_weights(weights, mapper=mapper)
|
|
|
|
def get_attn_backend(self) -> type[AttentionBackend]:
|
|
return self.attn_backend
|
|
|
|
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
|
return FullAttentionSpec(
|
|
block_size=vllm_config.cache_config.block_size,
|
|
num_kv_heads=self.num_kv_heads,
|
|
head_size=self.head_dim,
|
|
head_size_v=self.head_dim,
|
|
dtype=self.kv_cache_torch_dtype,
|
|
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
|
)
|
|
|
|
@eager_break_during_capture
|
|
def _run_qsa(
|
|
self,
|
|
projected_qk: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
query: torch.Tensor | None,
|
|
key: torch.Tensor | None,
|
|
value: torch.Tensor | None,
|
|
output: torch.Tensor,
|
|
output_gate: torch.Tensor | None,
|
|
qkv: torch.Tensor,
|
|
) -> None:
|
|
# query/key/value/output_gate are None when the fused prepare runs
|
|
# inside the indexer launch.
|
|
metadata = get_forward_context().attn_metadata
|
|
if isinstance(metadata, list):
|
|
metadata = metadata[0]
|
|
if not isinstance(metadata, dict):
|
|
output.zero_()
|
|
return
|
|
main_metadata = cast(FlashAttentionMetadata, metadata[self.layer_name])
|
|
if self.kv_cache.numel() == 0:
|
|
raise RuntimeError("QSA main K/V cache is not bound")
|
|
|
|
num_tokens = main_metadata.num_actual_tokens
|
|
side_metadata = cast(
|
|
QSAForwardMetadata,
|
|
metadata[self.indexer.raw_key_cache.prefix],
|
|
)
|
|
if side_metadata.num_actual_tokens != num_tokens:
|
|
raise RuntimeError("QSA main and side metadata token counts disagree")
|
|
selected, main_outputs = self.indexer(
|
|
projected_qk,
|
|
positions,
|
|
self.topk_indices_buffer[:num_tokens],
|
|
attn=self,
|
|
qkv=qkv,
|
|
slot_mapping=main_metadata.slot_mapping,
|
|
)
|
|
if selected.shape != (num_tokens, self.indexer.packed_output_width):
|
|
raise RuntimeError("QSA indexer returned an invalid selection shape")
|
|
impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl)
|
|
if main_outputs is None:
|
|
assert key is not None and value is not None
|
|
impl.do_kv_cache_update(
|
|
self,
|
|
key,
|
|
value,
|
|
self.kv_cache,
|
|
main_metadata.slot_mapping,
|
|
)
|
|
else:
|
|
query, output_gate = main_outputs
|
|
assert query is not None and output_gate is not None
|
|
impl.forward_qsa(
|
|
self,
|
|
query,
|
|
key,
|
|
value,
|
|
self.kv_cache,
|
|
main_metadata,
|
|
output,
|
|
token_to_req=side_metadata.token_to_req,
|
|
use_prefill_config=main_metadata.max_query_len > self._max_decode_query_len,
|
|
output_gate=output_gate,
|
|
)
|
|
|
|
def forward(
|
|
self,
|
|
positions: torch.Tensor,
|
|
hidden_states: torch.Tensor,
|
|
) -> torch.Tensor:
|
|
qkv, _ = self.qkv_proj(hidden_states)
|
|
if self.fuse_indexer_projection:
|
|
qkv, projected_qk = qkv.split(
|
|
[2 * self.q_size + 2 * self.kv_size, self.index_qk_size], dim=-1
|
|
)
|
|
else:
|
|
projected_qk, _ = self.indexer.index_qk_proj(hidden_states)
|
|
num_tokens = hidden_states.shape[0]
|
|
if not self.use_fused_qsa_prepare:
|
|
q, k, v, gate = self._project_qkv_gate(qkv, positions)
|
|
assert gate is not None
|
|
query = q.view(num_tokens, self.num_heads, self.head_dim)
|
|
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
|
|
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
|
|
else:
|
|
# Norm/RoPE/gate and the K/V cache write happen inside _run_qsa.
|
|
query = key = value = gate = None
|
|
attn_output = qkv.new_empty(num_tokens, self.num_heads, self.head_dim)
|
|
self._run_qsa(
|
|
projected_qk,
|
|
positions,
|
|
query,
|
|
key,
|
|
value,
|
|
attn_output,
|
|
gate,
|
|
qkv,
|
|
)
|
|
flat_output = attn_output.view(num_tokens, -1)
|
|
output, _ = self.o_proj(flat_output)
|
|
return output
|
|
|
|
|
|
__all__ = [
|
|
"QSAIndexer",
|
|
"Qwen4ExpQSAAttention",
|
|
"Qwen4ExpQSAFlashAttentionBackend",
|
|
"Qwen4ExpQSAFlashAttentionImpl",
|
|
]
|