Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com> Signed-off-by: lesj0610 <lesj0610@gmail.com> Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com>
545 lines
19 KiB
Python
545 lines
19 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""AMD ROCm QSA owner with Triton kernels."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import ClassVar, cast
|
|
|
|
import torch
|
|
from torch import nn
|
|
from transformers import Qwen4ExpTextConfig
|
|
|
|
from vllm.config import VllmConfig
|
|
from vllm.config.cache import CacheDType
|
|
from vllm.distributed import get_tensor_model_parallel_world_size
|
|
from vllm.forward_context import get_forward_context
|
|
from vllm.model_executor.layers.attention.attention import (
|
|
set_default_quant_scales,
|
|
)
|
|
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
|
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
|
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
|
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
|
from vllm.model_executor.layers.rotary_embedding import get_rope
|
|
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
|
from vllm.model_executor.models.qwen3_vl import mrope_positions_factor
|
|
from vllm.platforms import current_platform
|
|
from vllm.utils.torch_utils import (
|
|
LayerNameType,
|
|
_encode_layer_name,
|
|
_resolve_layer_name,
|
|
canonicalize_singleton_dim_strides,
|
|
direct_register_custom_op,
|
|
kv_cache_dtype_str_to_dtype,
|
|
)
|
|
from vllm.v1.attention.backend import (
|
|
AttentionBackend,
|
|
AttentionCGSupport,
|
|
AttentionType,
|
|
MultipleOf,
|
|
)
|
|
from vllm.v1.attention.backends.fa_utils import is_flash_attn_varlen_func_available
|
|
from vllm.v1.attention.backends.flash_attn import (
|
|
FlashAttentionBackend,
|
|
FlashAttentionImpl,
|
|
FlashAttentionMetadata,
|
|
FlashAttentionMetadataBuilder,
|
|
)
|
|
from vllm.v1.kv_cache_interface import (
|
|
FullAttentionSpec,
|
|
KVCacheSpec,
|
|
get_kv_quant_mode,
|
|
)
|
|
|
|
from ..common.qsa_cache import QSAForwardMetadata
|
|
from . import model
|
|
from .indexer_qsa import QSAIndexer
|
|
|
|
|
|
class Qwen4ExpQSAMetadataBuilder(FlashAttentionMetadataBuilder):
|
|
"""Flash metadata supporting uniform decode and target-verify graphs."""
|
|
|
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
|
|
|
|
|
class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
|
|
"""FullAttentionSpec backend used by the merged QSA owner."""
|
|
|
|
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
|
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"]
|
|
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "QWEN4_EXP_QSA_TRITON"
|
|
|
|
@staticmethod
|
|
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
|
# QSA consumes manager pages directly and does not use FA4 paged attention.
|
|
return [MultipleOf(16)]
|
|
|
|
@staticmethod
|
|
def get_impl_cls() -> type[Qwen4ExpQSAFlashAttentionImpl]:
|
|
return Qwen4ExpQSAFlashAttentionImpl
|
|
|
|
@staticmethod
|
|
def get_builder_cls() -> type[Qwen4ExpQSAMetadataBuilder]:
|
|
return Qwen4ExpQSAMetadataBuilder
|
|
|
|
@classmethod
|
|
def is_sparse(cls) -> bool:
|
|
return True
|
|
|
|
@classmethod
|
|
def supports_kv_connector(cls) -> bool:
|
|
return False
|
|
|
|
|
|
class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
|
"""Run paged sparse GQA with the QSA Triton kernel."""
|
|
|
|
supports_dcp: bool = False
|
|
supports_pcp: bool = False
|
|
|
|
def __init__(self, *args, **kwargs) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
if not is_flash_attn_varlen_func_available():
|
|
raise NotImplementedError("Qwen4Exp QSA requires FlashAttention")
|
|
if self.dcp_world_size != 1:
|
|
raise NotImplementedError(
|
|
"Qwen4Exp QSA does not support decode context parallelism"
|
|
)
|
|
if self.kv_cache_dtype not in ("auto", "bfloat16"):
|
|
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
|
|
self.supports_quant_query_input = False
|
|
|
|
def forward_qsa(
|
|
self,
|
|
layer: torch.nn.Module,
|
|
query: torch.Tensor,
|
|
key: torch.Tensor | None,
|
|
value: torch.Tensor | None,
|
|
kv_cache: torch.Tensor,
|
|
attn_metadata: FlashAttentionMetadata,
|
|
output: torch.Tensor,
|
|
token_to_req: torch.Tensor,
|
|
output_scale: torch.Tensor | None = None,
|
|
output_block_scale: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
del key, value
|
|
if output_scale is not None or output_block_scale is not None:
|
|
raise NotImplementedError("QSA does not support fused output quantization")
|
|
if self.alibi_slopes is not None or self.sinks is not None:
|
|
raise NotImplementedError("QSA does not support ALiBi or attention sinks")
|
|
if self.sliding_window != (-1, -1):
|
|
raise NotImplementedError("QSA does not support sliding-window attention")
|
|
|
|
num_tokens = attn_metadata.num_actual_tokens
|
|
output.zero_()
|
|
if num_tokens == 0:
|
|
return output
|
|
|
|
topk_buffer = getattr(layer, "topk_indices_buffer", None)
|
|
if topk_buffer is None:
|
|
raise RuntimeError("QSA owner did not provide its top-k buffer")
|
|
logical_indices = topk_buffer[:num_tokens]
|
|
token_to_req = token_to_req[:num_tokens]
|
|
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
|
key_cache = canonicalize_singleton_dim_strides(key_cache)
|
|
value_cache = canonicalize_singleton_dim_strides(value_cache)
|
|
if key_cache.dtype != torch.bfloat16 or query.dtype != torch.bfloat16:
|
|
raise NotImplementedError("Qwen4Exp QSA requires BF16 Q/K/V")
|
|
|
|
from .ops.qsa import qsa_sparse_paged_attention
|
|
|
|
qsa_sparse_paged_attention(
|
|
query[:num_tokens],
|
|
key_cache,
|
|
value_cache,
|
|
logical_indices,
|
|
attn_metadata.block_table,
|
|
token_to_req,
|
|
output[:num_tokens],
|
|
)
|
|
return output
|
|
|
|
|
|
class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
|
"""Merged Qwen full-attention owner with a QSA index side branch."""
|
|
|
|
supports_dcp = False
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
vllm_config: VllmConfig,
|
|
config: Qwen4ExpTextConfig,
|
|
layer_id: int,
|
|
quant_config: QuantizationConfig | None = None,
|
|
reduce_results: bool = True,
|
|
prefix: str = "",
|
|
) -> None:
|
|
nn.Module.__init__(self)
|
|
cache_config = vllm_config.cache_config
|
|
model_config = vllm_config.model_config
|
|
if cache_config is None:
|
|
raise ValueError("Qwen4Exp QSA requires a paged KV cache")
|
|
if model_config.dtype != torch.bfloat16:
|
|
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
|
|
if cache_config.cache_dtype not in ("auto", "bfloat16"):
|
|
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
|
|
if getattr(quant_config, "kv_cache_scheme", None) is not None:
|
|
raise NotImplementedError("Qwen4Exp QSA does not support KV quantization")
|
|
parallel_config = vllm_config.parallel_config
|
|
if (
|
|
parallel_config.prefill_context_parallel_size > 1
|
|
or parallel_config.decode_context_parallel_size > 1
|
|
):
|
|
raise NotImplementedError(
|
|
"Qwen4Exp QSA does not support context parallelism"
|
|
)
|
|
if not getattr(config, "is_causal", True):
|
|
raise NotImplementedError("Qwen4Exp QSA requires causal decoder attention")
|
|
|
|
self.config = config
|
|
self.hidden_size = int(config.hidden_size)
|
|
tp_size = get_tensor_model_parallel_world_size()
|
|
self.total_num_heads = int(config.num_attention_heads)
|
|
if self.total_num_heads % tp_size:
|
|
raise ValueError("QSA attention heads must be divisible by TP size")
|
|
self.num_heads = self.total_num_heads // tp_size
|
|
self.total_num_kv_heads = int(config.num_key_value_heads)
|
|
if self.total_num_kv_heads >= tp_size:
|
|
if self.total_num_kv_heads % tp_size:
|
|
raise ValueError("QSA KV heads must be divisible by TP size")
|
|
elif tp_size % self.total_num_kv_heads:
|
|
raise ValueError("TP size must be divisible by replicated QSA KV heads")
|
|
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
|
self.head_dim = int(config.head_dim or self.hidden_size // self.total_num_heads)
|
|
self.q_size = self.num_heads * self.head_dim
|
|
self.kv_size = self.num_kv_heads * self.head_dim
|
|
self.scaling = self.head_dim**-0.5
|
|
self.dual_chunk_attention_config = getattr(
|
|
config, "dual_chunk_attention_config", None
|
|
)
|
|
if self.dual_chunk_attention_config is not None:
|
|
raise NotImplementedError("Qwen4Exp QSA does not support dual-chunk RoPE")
|
|
# Qwen4Exp full-attention checkpoints always pack a sigmoid output
|
|
# gate next to Q, even when an inherited config default says otherwise.
|
|
self.attn_output_gate = True
|
|
|
|
self.qkv_proj = QKVParallelLinear(
|
|
self.hidden_size,
|
|
self.head_dim,
|
|
self.total_num_heads * (1 + self.attn_output_gate),
|
|
self.total_num_kv_heads,
|
|
bias=False,
|
|
quant_config=model.without_modelopt_fp4(quant_config),
|
|
prefix=f"{prefix}.qkv_proj",
|
|
)
|
|
self.o_proj = RowParallelLinear(
|
|
self.total_num_heads * self.head_dim,
|
|
self.hidden_size,
|
|
bias=False,
|
|
reduce_results=reduce_results,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.o_proj",
|
|
)
|
|
self.rotary_emb = get_rope(
|
|
head_size=self.head_dim,
|
|
max_position=config.max_position_embeddings,
|
|
rope_parameters=config.rope_parameters,
|
|
mrope_positions_factor=mrope_positions_factor(vllm_config),
|
|
)
|
|
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
|
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
|
|
|
mm_config = model_config.multimodal_config
|
|
text_only = mm_config is None or mm_config.language_model_only
|
|
self.use_fused_qk_norm_rope_gate = (
|
|
self.attn_output_gate
|
|
and getattr(self.rotary_emb, "is_neox_style", False)
|
|
and current_platform.is_cuda()
|
|
and text_only
|
|
)
|
|
|
|
self.layer_name = f"{prefix}.attn"
|
|
self.attn_type = AttentionType.DECODER
|
|
self.kv_cache_dtype = cache_config.cache_dtype
|
|
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
|
|
self.kv_cache_dtype, model_config
|
|
)
|
|
if self.kv_cache_torch_dtype != torch.bfloat16:
|
|
raise NotImplementedError("Qwen4Exp QSA requires BF16 cache storage")
|
|
self.kv_sharing_target_layer_name = None
|
|
self.kv_cache = torch.tensor([])
|
|
set_default_quant_scales(self, register_buffer=True)
|
|
|
|
self.attn_backend = Qwen4ExpQSAFlashAttentionBackend
|
|
self.impl = Qwen4ExpQSAFlashAttentionImpl(
|
|
self.num_heads,
|
|
self.head_dim,
|
|
self.scaling,
|
|
self.num_kv_heads,
|
|
None,
|
|
None,
|
|
self.kv_cache_dtype,
|
|
None,
|
|
AttentionType.DECODER,
|
|
None,
|
|
)
|
|
self.indexer = QSAIndexer(
|
|
vllm_config=vllm_config,
|
|
config=config,
|
|
layer_id=layer_id,
|
|
rotary_emb=self.rotary_emb,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.indexer",
|
|
)
|
|
self.use_fused_qsa_prepare = (
|
|
self.attn_output_gate
|
|
and self.indexer.use_fused_pre_indexer
|
|
and self.head_dim & (self.head_dim - 1) == 0
|
|
)
|
|
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
|
self.register_buffer(
|
|
"topk_indices_buffer",
|
|
torch.empty(
|
|
max_tokens,
|
|
self.indexer.output_width,
|
|
dtype=torch.int32,
|
|
),
|
|
persistent=False,
|
|
)
|
|
|
|
static_context = vllm_config.compilation_config.static_forward_context
|
|
if self.layer_name in static_context:
|
|
raise ValueError(f"Duplicate layer name: {self.layer_name}")
|
|
static_context[self.layer_name] = self
|
|
|
|
def get_attn_backend(self) -> type[AttentionBackend]:
|
|
return self.attn_backend
|
|
|
|
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
|
return FullAttentionSpec(
|
|
block_size=vllm_config.cache_config.block_size,
|
|
num_kv_heads=self.num_kv_heads,
|
|
head_size=self.head_dim,
|
|
head_size_v=self.head_dim,
|
|
dtype=self.kv_cache_torch_dtype,
|
|
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
|
)
|
|
|
|
def _run_qsa(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
query: torch.Tensor | None,
|
|
key: torch.Tensor | None,
|
|
value: torch.Tensor | None,
|
|
output: torch.Tensor,
|
|
qkv: torch.Tensor | None = None,
|
|
gate: torch.Tensor | None = None,
|
|
) -> None:
|
|
# query/key/value are None when the fused prepare runs inside the
|
|
# indexer launch, which then writes the gate into ``gate``.
|
|
metadata = get_forward_context().attn_metadata
|
|
if isinstance(metadata, list):
|
|
metadata = metadata[0]
|
|
if not isinstance(metadata, dict):
|
|
output.zero_()
|
|
if gate is not None:
|
|
gate.zero_()
|
|
return
|
|
main_metadata = cast(FlashAttentionMetadata, metadata[self.layer_name])
|
|
if self.kv_cache.numel() == 0:
|
|
raise RuntimeError("QSA main K/V cache is not bound")
|
|
|
|
num_tokens = main_metadata.num_actual_tokens
|
|
side_metadata = cast(
|
|
QSAForwardMetadata,
|
|
metadata[self.indexer.raw_key_cache.prefix],
|
|
)
|
|
if side_metadata.num_actual_tokens != num_tokens:
|
|
raise RuntimeError("QSA main and side metadata token counts disagree")
|
|
selected, main_outputs = self.indexer(
|
|
hidden_states,
|
|
positions,
|
|
self.topk_indices_buffer[:num_tokens],
|
|
attn=self,
|
|
qkv=qkv,
|
|
slot_mapping=main_metadata.slot_mapping,
|
|
gate_out=gate,
|
|
)
|
|
if selected.shape != (
|
|
num_tokens,
|
|
self.indexer.output_width,
|
|
):
|
|
raise RuntimeError("QSA indexer returned an invalid selection shape")
|
|
impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl)
|
|
if main_outputs is None:
|
|
assert key is not None and value is not None
|
|
impl.do_kv_cache_update(
|
|
self,
|
|
key,
|
|
value,
|
|
self.kv_cache,
|
|
main_metadata.slot_mapping,
|
|
)
|
|
else:
|
|
query, _ = main_outputs
|
|
# The gate is applied to every row outside this op; padded rows
|
|
# must stay finite so they keep the zero output forward_qsa gives.
|
|
assert gate is not None
|
|
gate[num_tokens:].zero_()
|
|
assert query is not None
|
|
impl.forward_qsa(
|
|
self,
|
|
query,
|
|
key,
|
|
value,
|
|
self.kv_cache,
|
|
main_metadata,
|
|
output,
|
|
token_to_req=side_metadata.token_to_req,
|
|
)
|
|
|
|
def forward(
|
|
self,
|
|
positions: torch.Tensor,
|
|
hidden_states: torch.Tensor,
|
|
) -> torch.Tensor:
|
|
qkv, _ = self.qkv_proj(hidden_states)
|
|
num_tokens = hidden_states.shape[0]
|
|
encoded_layer_name = _encode_layer_name(self.layer_name)
|
|
if self.use_fused_qsa_prepare:
|
|
# Norm/RoPE/gate and the K/V cache write happen inside the op.
|
|
attn_output = qkv.new_empty(num_tokens, self.num_heads, self.head_dim)
|
|
gate = torch.empty_like(attn_output)
|
|
if current_platform.opaque_attention_op():
|
|
torch.ops.vllm.qwen4_exp_qsa_prepare_with_output(
|
|
hidden_states,
|
|
positions,
|
|
qkv,
|
|
attn_output,
|
|
gate,
|
|
encoded_layer_name,
|
|
)
|
|
else:
|
|
qwen4_exp_qsa_prepare_with_output(
|
|
hidden_states,
|
|
positions,
|
|
qkv,
|
|
attn_output,
|
|
gate,
|
|
encoded_layer_name,
|
|
)
|
|
flat_output = attn_output.view(num_tokens, -1)
|
|
flat_output = flat_output * torch.sigmoid(gate.view(num_tokens, -1))
|
|
output, _ = self.o_proj(flat_output)
|
|
return output
|
|
|
|
q, k, v, gate = self._project_qkv_gate(qkv, positions)
|
|
query = q.view(num_tokens, self.num_heads, self.head_dim)
|
|
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
|
|
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
|
|
attn_output = torch.empty_like(query)
|
|
if current_platform.opaque_attention_op():
|
|
torch.ops.vllm.qwen4_exp_qsa_with_output(
|
|
hidden_states,
|
|
positions,
|
|
query,
|
|
key,
|
|
value,
|
|
attn_output,
|
|
encoded_layer_name,
|
|
)
|
|
else:
|
|
qwen4_exp_qsa_with_output(
|
|
hidden_states,
|
|
positions,
|
|
query,
|
|
key,
|
|
value,
|
|
attn_output,
|
|
encoded_layer_name,
|
|
)
|
|
flat_output = attn_output.view(num_tokens, -1)
|
|
if gate is not None:
|
|
flat_output = flat_output * torch.sigmoid(gate)
|
|
output, _ = self.o_proj(flat_output)
|
|
return output
|
|
|
|
|
|
def qwen4_exp_qsa_with_output(
|
|
hidden_states: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
query: torch.Tensor,
|
|
key: torch.Tensor,
|
|
value: torch.Tensor,
|
|
output: torch.Tensor,
|
|
layer_name: LayerNameType,
|
|
) -> None:
|
|
"""Run the complete QSA state/update/attend transaction."""
|
|
layer_name = _resolve_layer_name(layer_name)
|
|
layer = get_forward_context().no_compile_layers[layer_name]
|
|
if not isinstance(layer, Qwen4ExpQSAAttention):
|
|
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
|
|
layer._run_qsa(
|
|
hidden_states,
|
|
positions,
|
|
query,
|
|
key,
|
|
value,
|
|
output,
|
|
)
|
|
|
|
|
|
direct_register_custom_op(
|
|
op_name="qwen4_exp_qsa_with_output",
|
|
op_func=qwen4_exp_qsa_with_output,
|
|
mutates_args=["output"],
|
|
)
|
|
|
|
|
|
def qwen4_exp_qsa_prepare_with_output(
|
|
hidden_states: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
qkv: torch.Tensor,
|
|
output: torch.Tensor,
|
|
gate: torch.Tensor,
|
|
layer_name: LayerNameType,
|
|
) -> None:
|
|
"""Run the QSA transaction with the main QK-norm/RoPE/gate fused in."""
|
|
layer_name = _resolve_layer_name(layer_name)
|
|
layer = get_forward_context().no_compile_layers[layer_name]
|
|
if not isinstance(layer, Qwen4ExpQSAAttention):
|
|
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
|
|
layer._run_qsa(
|
|
hidden_states,
|
|
positions,
|
|
None,
|
|
None,
|
|
None,
|
|
output,
|
|
qkv=qkv,
|
|
gate=gate,
|
|
)
|
|
|
|
|
|
direct_register_custom_op(
|
|
op_name="qwen4_exp_qsa_prepare_with_output",
|
|
op_func=qwen4_exp_qsa_prepare_with_output,
|
|
mutates_args=["output", "gate"],
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"QSAIndexer",
|
|
"Qwen4ExpQSAAttention",
|
|
"Qwen4ExpQSAFlashAttentionBackend",
|
|
"Qwen4ExpQSAFlashAttentionImpl",
|
|
"qwen4_exp_qsa_prepare_with_output",
|
|
"qwen4_exp_qsa_with_output",
|
|
]
|