[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers

- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs,
  moriio, hisparse, example and hidden-states connectors
- Reduce weight_transfer to the base API used by model loading
- Remove kv_offload tiering, simple_kv_offload, ec cpu/example,
  snapshot command, engine snapshot server
- Remove transformers model fallback, dsv4 tokenizers, dead platforms
  cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn
  ops, dead layers (short_conv, fused_embed_norm, indexer_topk,
  relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar
- Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant
- Rebuild parser hubs around Qwen3 adapters only (36 dead adapters)
- Remove dead tests with the code they covered

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 02:19:03 +07:00
co-authored by Hinata
parent 54a396586e
commit 2b544b7ba0
353 changed files with 29 additions and 121269 deletions
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,205 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HyperConnection (Gated Residual) utilities for the AMD model variant.
Implements the HyperConnection residual scheme proposed in
"HyperConnections" (https://arxiv.org/abs/2409.19606). This AMD variant
delays each HC combine to the following HC mix boundary. HC glue kernels,
including fused combine+RMSNorm, live in ``ops/hc.py``; projections remain
standard vLLM Linear modules.
Hidden states between layers have shape ``[..., HC*HS]`` with HS inner
(HC outer, HS inner — checkpoint-native layout).
Typical usage inside a transformer decoder layer::
self.attn_hc = GatedResidual(hc_config)
hidden_states, block_input, injection = self.attn_hc.mix(hidden_states)
attention_output = attention(block_input)
hidden_states, block_input, injection = self.mlp_hc.combine_and_mix(
hidden_states, attention_output, injection
)
"""
import torch
from torch import nn
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
ReplicatedLinear,
)
from vllm.model_executor.models.utils import maybe_prefix
from ..common.hyperconnection import (
GroupedGemmaRMSNorm,
HyperConnectionConfig,
)
from .ops.hc import (
grouped_gemma_rmsnorm,
hc_combine,
hc_combine_norm,
hc_gate_mix,
hc_silu,
)
# ---------------------------------------------------------------------------
# Gated-residual variant
# ---------------------------------------------------------------------------
class GatedResidual(nn.Module):
"""Gated HyperConnection with learnable low-rank mixing and injection.
``combine_and_mix()`` runs the pre pipeline (grouped GemmaRMSNorm -> merged
low-rank down+inject GEMM -> silu -> up GEMM -> sigmoid -> gated mean
over the HC streams). When passed a pending block output and an injection,
it fuses their residual combine with the RMSNorm. Final mixers use
``use_combine=False`` and do not produce a new injection.
Weights: the norm owns the grouped GemmaRMSNorm affine; the projections
are vLLM Linear modules (merged replicated linear for down+inject), so
GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the
standard quant_method mechanism.
"""
def __init__(
self,
config: HyperConnectionConfig,
use_combine: bool = True,
prefix: str = "",
) -> None:
super().__init__()
self.config = config
self.lora_rank = config.hc_lowrank
self.hc_count = config.hc_count
self.hidden_size = config.hidden_size
self.use_combine = use_combine
norm_size = (
self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size
)
group_size = config.hidden_size if config.hc_per_branch_norm else None
# Normalize each H-sized HC stream independently while retaining a
# separate affine weight for every element of the HC*H layout.
self.hc_norm = GroupedGemmaRMSNorm(
norm_size,
eps=config.rms_norm_eps,
group_size=group_size,
dtype=config.params_dtype,
)
# -- vLLM Linear weights --------------------------------------------
# The merged skinny-GEMM shape is physically padded to 16 rows for
# alignment and efficient backend dispatch.
self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0
if use_combine:
self.input_mix_weight_down_block_inject = MergedColumnParallelLinear(
self.hyper_hidden_size,
[self.lora_rank, self.hc_count]
+ ([self.pad_size] if self.pad_size else []),
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_down_block_inject"),
return_bias=False,
disable_tp=True,
)
else:
self.input_mix_weight_down = ReplicatedLinear(
self.hyper_hidden_size,
self.lora_rank,
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_down"),
return_bias=False,
)
self.input_mix_weight_up = ReplicatedLinear(
self.lora_rank,
self.hyper_hidden_size,
bias=False,
params_dtype=config.params_dtype,
quant_config=None,
prefix=maybe_prefix(prefix, "input_mix_weight_up"),
return_bias=False,
)
def mix(
self, hidden_states: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
xn = grouped_gemma_rmsnorm(
hidden_states,
self.hc_norm.weight,
self.config.rms_norm_eps,
self.hc_count,
)
if self.use_combine:
# produce injection logits for combine
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
down_and_injection = self.input_mix_weight_down_block_inject(xn)
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
else:
lora = self.input_mix_weight_down(xn)
injection = None
lora = hc_silu(lora, self.hc_count)
gate = self.input_mix_weight_up(lora) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
def combine_and_mix(
self,
hidden_states: torch.Tensor,
prev_block_output: torch.Tensor,
prev_injection: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
"""Consume a pending combine, then prepare the next block input.
``hidden_states`` is the multi-stream state from before the pending
block's mix. Its combine with ``block_output`` is fused with this
module's input RMSNorm.
"""
hidden_states, xn = hc_combine_norm(
hidden_states,
prev_block_output,
prev_injection,
self.hc_norm.weight,
self.config.rms_norm_eps,
self.hc_count,
)
if self.use_combine:
# produce injection logits for combine
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
down_and_injection = self.input_mix_weight_down_block_inject(xn)
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
else:
lora = self.input_mix_weight_down(xn)
injection = None
lora = hc_silu(lora, self.hc_count)
gate = self.input_mix_weight_up(lora) # [M, D]
block_input = hc_gate_mix(xn, gate, self.hc_count)
return hidden_states, block_input, injection
def combine(
self,
hidden_states: torch.Tensor,
block_output: torch.Tensor,
injection: torch.Tensor,
) -> torch.Tensor:
return hc_combine(hidden_states, block_output, injection, self.hc_count)
@property
def hyper_hidden_size(self) -> int:
return self.hc_count * self.hidden_size
__all__ = [
"GatedResidual",
"GroupedGemmaRMSNorm",
"HyperConnectionConfig",
]
-420
View File
@@ -1,420 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen4Exp weight-free QSA indexer."""
from __future__ import annotations
from typing import TYPE_CHECKING, cast
import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.config import VllmConfig
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
from ..common.qsa_cache import (
QSACompressedKeyCache,
QSAForwardMetadata,
QSAKeyStateCache,
canonical_qsa_rope_positions,
)
from .ops.qsa_prepare import qsa_prepare, supports_fused_pre_indexer
if TYPE_CHECKING:
from .qsa import Qwen4ExpQSAAttention
def apply_qsa_rope(
rotary_emb: nn.Module,
positions: torch.Tensor,
tensor: torch.Tensor,
) -> torch.Tensor:
"""Apply the main attention's exact 1D/MRoPE composition to QSA heads."""
num_tokens, _, head_dim = tensor.shape
rotary_dim = rotary_emb.rotary_dim
cache = rotary_emb._match_cos_sin_cache_dtype(tensor) # noqa: SLF001
cos_sin = cache[positions]
cos, sin = cos_sin.chunk(2, dim=-1)
if positions.ndim == 2:
shape = tensor.shape
tensor, _ = triton_mrope(
tensor.reshape(num_tokens, -1),
tensor.new_empty((num_tokens, head_dim)),
cos,
sin,
rotary_emb.mrope_section,
head_dim,
rotary_dim,
rotary_emb.mrope_interleaved,
rotary_emb.is_neox_style,
)
return tensor.reshape(shape)
rotated = rotary_emb.apply_rotary_emb(
tensor[..., :rotary_dim],
cos,
sin,
)
return torch.cat((rotated, tensor[..., rotary_dim:]), dim=-1)
def apply_qsa_rmsnorm(
norm: GemmaRMSNorm,
tensor: torch.Tensor,
) -> torch.Tensor:
"""Use vLLM's portable RMSNorm implementation on ROCm."""
return cast(torch.Tensor, norm(tensor))
class QSAIndexer(nn.Module):
"""Replicated Q/K projection plus paged, weight-free QSA selection.
``prefix`` must be the checkpoint's indexer prefix, normally
``model.layers.N.self_attn.indexer``. Consequently the trainable names are
``index_qk_proj``, ``q_layernorm`` and ``k_layernorm`` under that prefix.
"""
def __init__(
self,
*,
vllm_config: VllmConfig,
config: Qwen4ExpTextConfig,
layer_id: int,
rotary_emb: nn.Module,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
if vllm_config.cache_config is None:
raise ValueError("QSA requires a paged KV cache")
if vllm_config.model_config.dtype != torch.bfloat16:
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
self.layer_id = int(layer_id)
self.index_n_heads = int(config.indexer_n_heads)
self.index_kv_heads = int(config.indexer_kv_heads)
self.index_head_dim = int(config.indexer_head_dim)
self.token_topk = int(config.indexer_budget)
self.compress_ratio = int(config.indexer_compress_ratio)
self.rotary_emb = rotary_emb
self.use_fused_pre_indexer = supports_fused_pre_indexer(
rotary_emb,
self.index_head_dim,
self.index_kv_heads,
self.compress_ratio,
)
self.prefix = prefix
# MTP step 0 selects the target-aligned rows; later steps reuse them
# while continuing to update the QSA side cache.
self.skip_topk = False
self.index_qk_proj = ReplicatedLinear(
int(config.hidden_size),
(self.index_n_heads + self.index_kv_heads) * self.index_head_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.index_qk_proj" if prefix else "index_qk_proj",
)
self.q_layernorm = GemmaRMSNorm(
self.index_head_dim,
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
)
self.k_layernorm = GemmaRMSNorm(
self.index_head_dim,
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
)
cache_config = vllm_config.cache_config
cache_prefix = f"{prefix}." if prefix else ""
self.raw_key_cache = QSAKeyStateCache(
head_size=self.index_head_dim,
dtype=torch.bfloat16,
cache_rope_positions=vllm_config.model_config.uses_mrope,
prefix=f"{cache_prefix}raw_key_cache",
cache_config=cache_config,
compress_ratio=self.compress_ratio,
vllm_config=vllm_config,
)
self.compressed_key_cache = QSACompressedKeyCache(
head_size=self.index_head_dim,
dtype=torch.bfloat16,
compress_ratio=self.compress_ratio,
prefix=f"{cache_prefix}compressed_key_cache",
cache_config=cache_config,
vllm_config=vllm_config,
)
@property
def output_width(self) -> int:
return self.token_topk + self.compress_ratio - 1
def project(
self,
hidden_states: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Project replicated Q/K and split, leaving both operands unrotated."""
qk, _ = self.index_qk_proj(hidden_states)
return qk.split(
(
self.index_n_heads * self.index_head_dim,
self.index_kv_heads * self.index_head_dim,
),
dim=-1,
)
def project_qk(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Project replicated Q/K, normalize+rotate Q, and preserve raw K."""
q_raw, token_k = self.project(hidden_states)
q = q_raw.reshape(-1, self.index_n_heads, self.index_head_dim)
q = apply_qsa_rmsnorm(
self.q_layernorm,
q.reshape(-1, self.index_head_dim),
).reshape_as(q)
q = apply_qsa_rope(self.rotary_emb, positions, q)
return q, token_k.reshape(-1, 1, self.index_head_dim)
def normalize_compressed_keys(
self,
compressed_keys: torch.Tensor,
first_rope_positions: torch.Tensor,
) -> torch.Tensor:
"""Normalize pooled K and apply the first token's exact group position."""
keys = compressed_keys.reshape(-1, self.index_head_dim)
keys = apply_qsa_rmsnorm(self.k_layernorm, keys).reshape(
-1, 1, self.index_head_dim
)
if getattr(self.rotary_emb, "mrope_section", None):
positions = first_rope_positions.transpose(0, 1)
else:
positions = first_rope_positions[:, 0]
return apply_qsa_rope(self.rotary_emb, positions, keys)
def _metadata(
self,
) -> tuple[QSAForwardMetadata, QSAForwardMetadata] | None:
metadata = get_forward_context().attn_metadata
if isinstance(metadata, list):
metadata = metadata[0]
if not isinstance(metadata, dict):
return None
raw = cast(QSAForwardMetadata, metadata[self.raw_key_cache.prefix])
compressed = cast(
QSAForwardMetadata, metadata[self.compressed_key_cache.prefix]
)
if raw.num_actual_tokens != compressed.num_actual_tokens:
raise RuntimeError("QSA side-cache metadata token counts disagree")
if not raw.logical_positions.is_cuda and (
not torch.equal(raw.logical_positions, compressed.logical_positions)
):
raise RuntimeError("QSA side-cache metadata positions disagree")
return raw, compressed
def _update_and_compress(
self,
token_k: torch.Tensor,
positions: torch.Tensor,
raw_metadata: QSAForwardMetadata,
compressed_metadata: QSAForwardMetadata,
) -> None:
num_tokens = raw_metadata.num_actual_tokens
raw_key_cache = self.raw_key_cache.key_cache
rope_position_cache = self.raw_key_cache.rope_position_cache
from .ops.qsa import qsa_compress_groups_with_ratio, qsa_store_cache_rows
if rope_position_cache is None:
position_rows = raw_metadata.logical_positions.view(-1, 1, 1).expand(
-1, 1, 3
)
else:
position_rows = canonical_qsa_rope_positions(positions)[:num_tokens].to(
device=raw_key_cache.device
)
pooled, first_positions = qsa_compress_groups_with_ratio(
token_k[:num_tokens],
position_rows,
raw_key_cache,
raw_metadata.block_table,
raw_metadata.token_to_req,
raw_metadata.query_start_loc,
raw_metadata.logical_positions,
compressed_metadata.slot_mapping,
self.compress_ratio,
rope_position_cache,
)
normalized = self.normalize_compressed_keys(pooled, first_positions)
qsa_store_cache_rows(
self.compressed_key_cache.kv_cache,
compressed_metadata.slot_mapping,
normalized,
)
qsa_store_cache_rows(
raw_key_cache,
raw_metadata.slot_mapping,
token_k[:num_tokens],
)
if rope_position_cache is not None:
qsa_store_cache_rows(
rope_position_cache,
raw_metadata.slot_mapping,
position_rows,
)
def _fused_prepare(
self,
projected_q: torch.Tensor,
raw_keys: torch.Tensor,
positions: torch.Tensor,
raw_metadata: QSAForwardMetadata,
compressed_metadata: QSAForwardMetadata,
attn: Qwen4ExpQSAAttention,
qkv: torch.Tensor,
slot_mapping: torch.Tensor,
gate_out: torch.Tensor | None,
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
"""Prepare the indexer and ``attn``'s main attention in one launch.
Normalizes and rotates the index Q, compresses K and writes both side
caches, which replaces ``project_qk``'s norm/RoPE half and the whole of
``_update_and_compress``. The same launch applies ``attn``'s QK-norm and
RoPE, copies its gate and writes its K/V into ``attn.kv_cache``,
replacing ``_project_qkv_gate`` and ``do_kv_cache_update``. Both halves
must stay numerically interchangeable with what they replace.
"""
num_tokens = raw_metadata.num_actual_tokens
raw_key_cache = self.raw_key_cache
q = projected_q.new_empty(num_tokens, self.index_n_heads, self.index_head_dim)
main_outputs = qsa_prepare(
projected_q,
raw_keys,
positions,
self.rotary_emb.cos_sin_cache,
self.q_layernorm.weight,
self.k_layernorm.weight,
self.q_layernorm.variance_epsilon,
q,
raw_key_cache.kv_cache,
raw_metadata.slot_mapping,
raw_metadata.block_table,
raw_metadata.query_start_loc,
raw_metadata.logical_positions,
self.compressed_key_cache.kv_cache,
compressed_metadata.slot_mapping,
compressed_metadata.k_work_metadata,
compress_ratio=self.compress_ratio,
mrope_section=getattr(self.rotary_emb, "mrope_section", None),
rope_pos_offset=(
raw_key_cache.rope_position_offset
if raw_key_cache.cache_rope_positions
else None
),
main_qkv=qkv[:num_tokens],
main_q_norm_weight=attn.q_norm.weight,
main_k_norm_weight=attn.k_norm.weight,
main_eps=attn.q_norm.variance_epsilon,
main_kv_cache=attn.kv_cache.transpose(1, 2),
main_slot_mapping=slot_mapping[:num_tokens],
main_k_scale=attn._k_scale_float,
main_v_scale=attn._v_scale_float,
main_gate_out=None if gate_out is None else gate_out[:num_tokens],
)
return q, main_outputs
def _select(
self,
q: torch.Tensor,
metadata: QSAForwardMetadata,
out: torch.Tensor | None,
) -> torch.Tensor:
from .ops.qsa import qsa_select_paged_tokens
return qsa_select_paged_tokens(
q,
self.compressed_key_cache.kv_cache,
metadata.block_table,
metadata.token_to_req,
metadata.logical_positions,
metadata.seq_lens,
self.token_topk,
self.compress_ratio,
out,
)
def forward(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
out: torch.Tensor | None = None,
*,
attn: Qwen4ExpQSAAttention,
qkv: torch.Tensor | None = None,
slot_mapping: torch.Tensor | None = None,
gate_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]:
"""Return fixed-width request-relative token indices padded with ``-1``.
With ``attn.use_fused_qsa_prepare``, the same launch also writes
``attn``'s K/V into ``attn.kv_cache`` at ``slot_mapping`` and prepares
its Q and gate from ``qkv``, returned as the second element (None
otherwise). The gate is written into ``gate_out`` when given. ``qkv``,
``slot_mapping`` and ``gate_out`` are only read in that mode.
"""
metadata = self._metadata()
if metadata is None:
# Preserve step-0 indices when later MTP steps reuse the buffer.
if self.skip_topk and out is not None:
return out, None
result = torch.full(
(hidden_states.shape[0], self.output_width),
-1,
dtype=torch.int32,
device=hidden_states.device,
)
if out is not None:
out.copy_(result)
return out, None
return result, None
raw_metadata, compressed_metadata = metadata
num_tokens = raw_metadata.num_actual_tokens
hidden_states = hidden_states[:num_tokens]
positions = positions[..., :num_tokens]
main_outputs: tuple[torch.Tensor, torch.Tensor] | None = None
if attn.use_fused_qsa_prepare:
if qkv is None or slot_mapping is None:
raise ValueError("fused QSA prepare requires qkv and slot_mapping")
projected_q, raw_keys = self.project(hidden_states)
q, main_outputs = self._fused_prepare(
projected_q,
raw_keys,
positions,
raw_metadata,
compressed_metadata,
attn,
qkv,
slot_mapping,
gate_out,
)
else:
q, token_k = self.project_qk(hidden_states, positions)
self._update_and_compress(
token_k,
positions,
raw_metadata,
compressed_metadata,
)
if self.skip_topk:
if out is None:
raise RuntimeError("QSA top-k reuse requires an output buffer")
return out, main_outputs
return self._select(q, compressed_metadata, out), main_outputs
__all__ = ["QSAIndexer", "apply_qsa_rope"]
@@ -1,14 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen4Exp low-latency GEMM hook for AMD ROCm."""
import torch
from torch import nn
def enable_qwen4_exp_low_latency_gemm(
module: nn.Module,
dtype: torch.dtype,
) -> None:
"""Keep the standard vLLM linear methods on AMD ROCm."""
del module, dtype
File diff suppressed because it is too large Load Diff
-143
View File
@@ -1,143 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Model-runner state for Qwen4Exp PLE inputs."""
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
from vllm.v1.worker.gpu.states import RequestState
class Qwen4ExpModelState(MambaHybridModelState):
"""Add rollback-safe PLE n-gram context to the model inputs."""
def __init__(
self,
vllm_config: VllmConfig,
model: nn.Module,
encoder_cache: EncoderCache | None,
device: torch.device,
) -> None:
super().__init__(vllm_config, model, encoder_cache, device)
config = self.model_config.hf_text_config
self.uses_ngram_embedding = bool(config.ple_layer_ids)
if not self.uses_ngram_embedding:
self.ngram_context_len = 0
self.ngram_eos_token_id = 0
return
if vllm_config.parallel_config.pipeline_parallel_size > 1:
raise RuntimeError(
"N-gram PLE embedding currently requires "
"pipeline_parallel_size=1 because non-first pipeline ranks do "
"not receive the raw input_ids required by PLE. Please run "
"with PP=1."
)
self.ngram_context_len = int(config.ngram_size) - 1
if self.ngram_context_len <= 0:
raise ValueError("N-gram embedding requires context length >= 1.")
self.ngram_eos_token_id = int(config.eos_token_id)
self.ngram_context = torch.full(
(self.max_num_reqs, self.ngram_context_len),
self.ngram_eos_token_id,
dtype=torch.int32,
device=self.device,
)
self.ngram_context_offsets = torch.arange(
-self.ngram_context_len,
0,
dtype=torch.int64,
device=self.device,
)
self.ple_query_start_loc = torch.zeros(
self.max_num_reqs + 1,
dtype=torch.int32,
device=self.device,
)
def _prepare_ngram_context(
self,
input_batch: InputBatch,
req_states: RequestState,
) -> torch.Tensor:
num_reqs = input_batch.num_reqs
num_reqs_padded = input_batch.num_reqs_after_padding
context = self.ngram_context[:num_reqs_padded]
context.fill_(self.ngram_eos_token_id)
if num_reqs == 0:
return context
request_indices = input_batch.idx_mapping[:num_reqs].long()
context_end = req_states.num_computed_tokens.gpu[request_indices].long()
token_indices = context_end.unsqueeze(1) + self.ngram_context_offsets
valid_tokens = token_indices >= 0
token_indices.clamp_min_(0)
context_tokens = req_states.all_token_ids.gpu[
request_indices.unsqueeze(1), token_indices
]
context[:num_reqs].copy_(
torch.where(
valid_tokens,
context_tokens,
context_tokens.new_full((), self.ngram_eos_token_id),
)
)
return context
def prepare_inputs(
self,
input_batch: InputBatch,
req_states: RequestState,
) -> dict[str, Any]:
model_inputs = super().prepare_inputs(input_batch, req_states)
if not self.uses_ngram_embedding:
return model_inputs
num_reqs_padded = input_batch.num_reqs_after_padding
query_start_loc = self.ple_query_start_loc[: num_reqs_padded + 1]
query_start_loc.copy_(input_batch.query_start_loc[: num_reqs_padded + 1])
model_inputs.update(
query_start_loc=query_start_loc,
ngram_context=self._prepare_ngram_context(input_batch, req_states),
)
return model_inputs
def prepare_dummy_inputs(
self,
num_reqs: int,
num_tokens: int,
) -> dict[str, Any]:
model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens)
if not self.uses_ngram_embedding:
return model_inputs
query_start_loc = self.ple_query_start_loc[: num_reqs + 1]
query_start_loc[0] = 0
tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs)
query_lens = torch.full(
(num_reqs,),
tokens_per_req,
dtype=query_start_loc.dtype,
device=query_start_loc.device,
)
if num_extra_tokens > 0:
query_lens[-num_extra_tokens:] += 1
torch.cumsum(query_lens, dim=0, out=query_start_loc[1:])
ngram_context = self.ngram_context[:num_reqs]
ngram_context.fill_(self.ngram_eos_token_id)
model_inputs.update(
query_start_loc=query_start_loc,
ngram_context=ngram_context,
)
return model_inputs
__all__ = ["Qwen4ExpModelState"]
-495
View File
@@ -1,495 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Qwen4Exp MTP (Multi-Token Predictor) model.
The MTP draft model reuses the Qwen4Exp backbone (PLE/HC/MoE) but:
- drops all multi-modal handling (text-only),
- forces PLE off while keeping the main model's HC stream count,
- fuses the backbone hidden and the new-token embedding via
``residual_linear_shared`` (fc_embedding + shared fc_hidden) instead of
the ``Linear(2H, H)`` + repeat used by other MTP variants,
- emits TWO hidden streams per step (scheme A): a single stream [T, H]
(final-mixer collapsed, fed to the LM head) and a pre-final-mixer
multi stream [T, hc_count*H] (fed to the next draft step).
"""
from collections.abc import Iterable
import regex as re
import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig, replace, set_current_vllm_config
from vllm.distributed import get_pp_group
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.utils import configure_quant_config
from vllm.model_executor.models.interfaces import (
LocalArgmaxMixin,
SupportsHCSP,
SupportsPP,
)
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
get_draft_quant_config,
make_empty_intermediate_tensors_factory,
maybe_fuse_shared_experts,
maybe_prefix,
)
from vllm.models.common.ops.sequence_parallel import sp_all_gather, sp_shard
from vllm.sequence import IntermediateTensors
from ..common.hyperconnection import is_hc_sequence_parallel_enabled
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .model import (
_EXTRA_WEIGHTS_MAPPER,
_QWEN4_EXP_IGNORED_MISSING_SUFFIXES,
Qwen4ExpDecoderLayer,
Qwen4ExpMixtureOfExperts,
Qwen4ExpSparseMoeBlock,
)
def _remap_ignored_layers(
ignored_layers: list[str],
mtp_start_layer_idx: int,
) -> list[str]:
remapped: list[str] = []
for name in ignored_layers:
if name.startswith("mtp."):
new_name = re.sub(
r"(?<=\.layers\.)\d+",
lambda m: str(mtp_start_layer_idx + int(m.group(0))),
name,
)
remapped.append(new_name)
else:
remapped.append(name)
return remapped
def _remap_quantized_layers(
quantized_layers: dict[str, dict],
mtp_start_layer_idx: int,
) -> dict[str, dict]:
"""Map checkpoint MTP layer indices to standalone draft indices."""
return {
_remap_ignored_layers([name], mtp_start_layer_idx)[0]: layer_info
for name, layer_info in quantized_layers.items()
}
def _remap_mtp_weight_name(name: str) -> str | None:
"""Map Qwen4Exp checkpoint paths into the standalone draft model."""
for checkpoint_prefix in (
"model.language_model.",
"language_model.",
):
if name.startswith(checkpoint_prefix):
name = name.removeprefix(checkpoint_prefix)
break
if name.startswith("embed_tokens."):
name = f"model.{name}"
if name.startswith("model.mtp."):
name = name.removeprefix("model.")
if name.startswith("mtp.shared_head.head."):
return name.replace("mtp.shared_head.head.", "lm_head.", 1)
if name.startswith("model.shared_head.head."):
return name.replace("model.shared_head.head.", "lm_head.", 1)
if name.startswith("shared_head.head."):
return name.replace("shared_head.head.", "lm_head.", 1)
if name.startswith("model.lm_head."):
return name.removeprefix("model.")
if name.startswith("mtp."):
return name.replace("mtp.", "model.", 1)
if name.startswith("model.embed_tokens.") or name.startswith("lm_head."):
return name
return None
def _make_draft_vllm_config(
vllm_config: VllmConfig,
mtp_start_layer_idx: int,
) -> VllmConfig:
"""Ensure that the draft model config is set in the vLLM config."""
speculative_config = vllm_config.speculative_config
if speculative_config is None or speculative_config.draft_model_config is None:
raise ValueError("speculative_config.draft_model_config must be set")
draft_quant_config = get_draft_quant_config(vllm_config)
# inject packed and ignored modules to the quantization config of draft model
if draft_quant_config is not None:
configure_quant_config(draft_quant_config, Qwen4ExpMTP)
ignored_layers = getattr(draft_quant_config, "ignored_layers", None)
if ignored_layers:
setattr( # noqa: B010
draft_quant_config,
"ignored_layers",
_remap_ignored_layers(ignored_layers, mtp_start_layer_idx),
)
exclude_modules = getattr(draft_quant_config, "exclude_modules", None)
if exclude_modules:
setattr( # noqa: B010
draft_quant_config,
"exclude_modules",
_remap_ignored_layers(exclude_modules, mtp_start_layer_idx),
)
quantized_layers = getattr(draft_quant_config, "quantized_layers", None)
if quantized_layers:
setattr( # noqa: B010
draft_quant_config,
"quantized_layers",
_remap_quantized_layers(quantized_layers, mtp_start_layer_idx),
)
draft_vllm_config = replace(
vllm_config,
model_config=speculative_config.draft_model_config,
)
# VllmConfig post-init derives the target quant config, so restore the
# independently resolved draft quant config after replacement.
draft_vllm_config.quant_config = draft_quant_config
return draft_vllm_config
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
"hidden_states": 0,
}
)
class Qwen4ExpMultiTokenPredictor(nn.Module):
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
model_config = vllm_config.model_config
config: Qwen4ExpTextConfig = model_config.hf_text_config
self.config = config
# MoE SP also enables HC SP, while HC SP can be enabled independently.
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
self.vocab_size = config.vocab_size
self.mtp_start_layer_idx = config.num_hidden_layers
self.num_mtp_layers = getattr(config, "mtp_num_hidden_layers", 1)
self.hidden_size = config.hidden_size
self.hc_count = config.hc_count
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, self.hidden_size)
draft_vllm_config = _make_draft_vllm_config(
vllm_config,
self.mtp_start_layer_idx,
)
with set_current_vllm_config(draft_vllm_config, prefix=prefix):
# residual_linear_shared fusion: fc_embedding projects the token
# embedding, fc_hidden (shared across HC branches) projects the
# backbone hidden; the embedding is added as a residual to every
# branch (see mtp_residual_linear_shared.md).
self.fc_embedding = ColumnParallelLinear(
self.hidden_size,
self.hidden_size,
gather_output=True,
bias=False,
return_bias=False,
quant_config=draft_vllm_config.quant_config,
prefix=f"{prefix}.fc_embedding",
)
self.fc_hidden = ColumnParallelLinear(
self.hidden_size,
self.hidden_size,
gather_output=True,
bias=False,
return_bias=False,
quant_config=draft_vllm_config.quant_config,
prefix=f"{prefix}.fc_hidden",
)
self.layers = nn.ModuleList(
Qwen4ExpDecoderLayer(
draft_vllm_config,
layer_type="qwen_sparse_attention",
prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
)
for idx in range(self.num_mtp_layers)
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen4ExpSparseMoeBlock,
"mlp",
)
self.pre_fc_norm_embedding = GemmaRMSNorm(
self.hidden_size, eps=config.rms_norm_eps
)
self.pre_fc_norm_hidden = GemmaRMSNorm(
self.hidden_size * self.hc_count, eps=config.rms_norm_eps
)
# HC final mixer collapses the multi stream into [T, H] for the LM head.
hc_config = HyperConnectionConfig(
hc_count=config.hc_count,
hidden_size=config.hidden_size,
params_dtype=torch.bfloat16,
hc_lowrank=config.hc_lowrank,
rms_norm_eps=config.rms_norm_eps,
hc_per_branch_norm=True,
)
self.hyper_connection_mixer = GatedResidual(
hc_config,
use_combine=False,
prefix=maybe_prefix(prefix, "hyper_connection_mixer"),
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], self.hidden_size * self.hc_count
)
def _iter_qsa_attentions(self):
"""Yield MTP attention modules that own a QSA indexer."""
for layer in self.layers:
attention = getattr(layer, "self_attn", None)
if (
attention is not None
and getattr(attention, "indexer", None) is not None
):
yield attention
def set_skip_topk(self, skip: bool) -> None:
"""Select on MTP step 0 and reuse its QSA indices on later steps."""
for attention in self._iter_qsa_attentions():
attention.indexer.skip_topk = skip
def compact_topk_indices(self, row_indices: torch.Tensor) -> None:
"""Keep each request's target-aligned step-0 sparse-index row."""
num_rows = row_indices.numel()
for attention in self._iter_qsa_attentions():
buffer = attention.topk_indices_buffer
selected = buffer.index_select(0, row_indices)
buffer[:num_rows].copy_(selected)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor | None = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
hc_count = self.hc_count
hidden_size = self.hidden_size
if get_pp_group().is_first_rank:
assert hidden_states is not None
if inputs_embeds is None:
assert input_ids is not None
inputs_embeds = self.embed_input_ids(input_ids)
# Embedding branch: pre-norm -> fc_embedding -> [T, H].
inputs_embeds = self.pre_fc_norm_embedding(inputs_embeds)
inputs_embeds = self.fc_embedding(inputs_embeds)
# Backbone hidden is multi-stream [T, hc_count*H] (scheme A:
# the main model truly emits the pre-final-mixer multi stream
# on the first step; subsequent steps reuse the prior draft
# step's multi stream).
num_tokens = hidden_states.shape[0]
hidden_states = hidden_states.view(num_tokens, hc_count, hidden_size)
hidden_states = self.pre_fc_norm_hidden(hidden_states.flatten(-2)).view(
num_tokens, hc_count, hidden_size
)
hidden_states = self.fc_hidden(hidden_states)
# Add the embedding residual to every branch, then fold back
# to [T, hc_count*H] (HC outer, HS inner) for the HC decoder.
hidden_states = inputs_embeds.unsqueeze(-2) + hidden_states
hidden_states = hidden_states.flatten(-2)
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
current_step_idx = spec_step_idx % self.num_mtp_layers
full_num_tokens = positions.shape[-1]
if self.use_hc_sequence_parallel:
hidden_states = sp_shard(hidden_states)
layer = self.layers[current_step_idx]
hidden_states, block_output, injection = layer(
hidden_states=hidden_states,
prev_block_output=None,
prev_injection=None,
positions=positions,
input_ids=None,
query_start_loc=None,
ngram_context=None,
)
if not get_pp_group().is_last_rank:
# As in the target model, PP carries a materialized tensor rather
# than the delayed hidden/output/injection tuple.
hidden_states = layer.mlp_hyper_connection.combine(
hidden_states, block_output, injection
)
return IntermediateTensors({"hidden_states": hidden_states})
# Last PP rank finalize. Keep both:
# (A) sample_hidden_states [T, H] -> single stream for the LM head
# (B) multi_hidden [T, hc_count*H] -> pre-final-mixer multi stream
# for the next draft step (zero extra compute, just kept).
multi_hidden, sample_hidden_states, _ = (
self.hyper_connection_mixer.combine_and_mix(
hidden_states, block_output, injection
)
)
if self.use_hc_sequence_parallel:
# Gather both outputs together, then restore their contiguous layouts.
packed_hidden_states = torch.cat(
[sample_hidden_states, multi_hidden], dim=-1
)
packed_hidden_states = sp_all_gather(packed_hidden_states)[:full_num_tokens]
sample_hidden_states, multi_hidden = packed_hidden_states.split(
[hidden_size, hc_count * hidden_size], dim=-1
)
sample_hidden_states = sample_hidden_states.contiguous()
multi_hidden = multi_hidden.contiguous()
return sample_hidden_states, multi_hidden
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights,
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=getattr(self.config, "num_experts", 0) or 0,
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
)
mapper = self.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
)
loader = AutoWeightsLoader(
self,
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
)
return loader.load_weights(weights, mapper=mapper)
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
"hidden_states": 0,
}
)
class Qwen4ExpMTP(
LocalArgmaxMixin,
nn.Module,
SupportsHCSP,
SupportsPP,
Qwen4ExpMixtureOfExperts,
):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
"in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
"in_proj_ba": ["in_proj_b", "in_proj_a"],
"input_mix_weight_down_block_inject": [
"input_mix_weight_down",
"block_inject_weight",
"_input_mix_padding",
],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
self.quant_config = vllm_config.quant_config
super().__init__()
self.config = config
self.model = Qwen4ExpMultiTokenPredictor(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "mtp"),
)
if get_pp_group().is_last_rank:
if config.tie_word_embeddings:
self.lm_head = self.model.embed_tokens
else:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
self.set_moe_parameters(self.model.layers)
enable_qwen4_exp_low_latency_gemm(self, vllm_config.model_config.dtype)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor | None = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
return self.model(
input_ids,
positions,
hidden_states,
intermediate_tensors,
inputs_embeds,
spec_step_idx=spec_step_idx,
)
def compute_logits(
self, hidden_states: torch.Tensor, spec_step_idx: int = 0
) -> torch.Tensor | None:
return self.logits_processor(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
def remap_weight_names():
for name, weight in weights:
remapped_name = _remap_mtp_weight_name(name)
if remapped_name is not None:
yield remapped_name, weight
mapper = WeightsMapper(
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
)
loader = AutoWeightsLoader(
self,
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
)
return loader.load_weights(remap_weight_names(), mapper=mapper)
__all__ = ["Qwen4ExpMTP", "Qwen4ExpMultiTokenPredictor"]
@@ -1,3 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""AMD ROCm Qwen4Exp kernels; import leaf modules directly."""
-487
View File
@@ -1,487 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""AMD ROCm HyperConnection kernels for Qwen4Exp."""
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
@triton.jit
def _grouped_gemma_rmsnorm_kernel(
x_ptr,
w_ptr,
y_ptr,
stride_x,
stride_y,
DIM: tl.constexpr,
NUM_GROUPS: tl.constexpr,
W_SHARED: tl.constexpr,
EPS: tl.constexpr,
launch_pdl: tl.constexpr,
) -> None:
GROUP_DIM: tl.constexpr = DIM // NUM_GROUPS
BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(GROUP_DIM)
pid = tl.program_id(0)
group_id = pid % NUM_GROUPS
row = pid // NUM_GROUPS
offs_g = tl.arange(0, BLOCK_SIZE)
offsets = group_id * GROUP_DIM + offs_g
mask = offs_g < GROUP_DIM
# A [GROUP_DIM] affine is shared; a [DIM] affine follows the grouped
# checkpoint layout.
w_offs = offs_g if W_SHARED else offsets
if launch_pdl:
tl.extra.cuda.gdc_wait()
x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0).to(tl.float32)
w = tl.load(w_ptr + w_offs, mask, other=0.0)
rrms = tl.rsqrt(tl.sum(x * x) / GROUP_DIM + EPS)
# Gemma's (1 + w) affine is written this way to lower to an FMA.
y = x * rrms
y += y * w.to(tl.float32)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(y_ptr + row * stride_y + offsets, y, mask)
def _grouped_gemma_rmsnorm(
x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
) -> torch.Tensor:
N, DIM = x.shape
assert x.stride(1) == 1, "grouped Gemma RMSNorm requires unit inner stride"
assert weight.is_contiguous(), "grouped Gemma RMSNorm weight must be contiguous"
assert DIM % num_groups == 0
group_dim = DIM // num_groups
assert weight.numel() in (group_dim, DIM)
y = x.new_empty(x.shape)
_grouped_gemma_rmsnorm_kernel[(N * num_groups,)](
x,
weight,
y,
x.stride(0),
y.stride(0),
DIM,
num_groups,
W_SHARED=weight.numel() == group_dim,
EPS=eps,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return y
@triton.jit
def _hc_silu_kernel(
x_ptr,
y_ptr,
stride_x,
stride_y,
DIM: tl.constexpr,
HC: tl.constexpr,
launch_pdl: tl.constexpr,
) -> None:
BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(DIM)
row = tl.program_id(0)
offs = tl.arange(0, BLOCK_SIZE)
mask = offs < DIM
if launch_pdl:
tl.extra.cuda.gdc_wait()
x = tl.load(x_ptr + row * stride_x + offs, mask).to(tl.float32) / HC
y = x * tl.sigmoid(x)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(y_ptr + row * stride_y + offs, y, mask)
def _hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
num_tokens, DIM = x.shape
assert x.stride(1) == 1
output = x.new_empty(x.shape)
_hc_silu_kernel[(num_tokens,)](
x,
output,
x.stride(0),
output.stride(0),
DIM=DIM,
HC=hc_count,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return output
@triton.jit
def _hc_gate_mix_kernel(
x_ptr,
g_ptr,
y_ptr,
stride_x,
stride_g,
stride_y,
DIM: tl.constexpr,
HC: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
) -> None:
HC_DIM: tl.constexpr = DIM // HC
row = tl.program_id(0)
tile_id = tl.program_id(1)
offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offs_inner < HC_DIM
if launch_pdl:
tl.extra.cuda.gdc_wait()
# The constexpr loop is unrolled and keeps one stream live at a time.
# Materializing [HC, BLOCK_SIZE] more than doubles latency at large M.
acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
for stream in tl.static_range(HC):
offsets = stream * HC_DIM + offs_inner
g = tl.load(g_ptr + row * stride_g + offsets, mask, other=0.0)
x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0)
acc += tl.sigmoid(g.to(tl.float32)) * x.to(tl.float32)
acc /= HC
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(y_ptr + row * stride_y + offs_inner, acc, mask)
def _hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
N, DIM = gate.shape
assert x.shape == gate.shape
assert DIM % hc_count == 0
assert x.stride(1) == 1
assert gate.stride(1) == 1
HC_DIM = DIM // hc_count
out = x.new_empty(N, HC_DIM)
BLOCK_SIZE = 512
_hc_gate_mix_kernel[(N, triton.cdiv(HC_DIM, BLOCK_SIZE))](
x,
gate,
out,
x.stride(0),
gate.stride(0),
out.stride(0),
DIM,
hc_count,
BLOCK_SIZE,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return out
@triton.jit
def _hc_combine_kernel(
block_ptr,
res_ptr,
inj_ptr,
out_ptr,
stride_block,
stride_res,
stride_inj,
stride_out,
HC_DIM: tl.constexpr,
HC: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
) -> None:
HC_PAD: tl.constexpr = triton.next_power_of_2(HC)
row = tl.program_id(0)
tile_id = tl.program_id(1)
offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask_inner = offs_inner < HC_DIM
offs_hc = tl.arange(0, HC_PAD)
mask_hc = offs_hc < HC
offs = offs_hc[:, None] * HC_DIM + offs_inner[None, :]
mask = mask_hc[:, None] & mask_inner[None, :]
if launch_pdl:
tl.extra.cuda.gdc_wait()
inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)
res = tl.load(res_ptr + row * stride_res + offs, mask, other=0.0)
# Keeping HC as a broadcast dimension is faster here than four separate
# residual load/store sequences.
inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
out = res.to(tl.float32) + block.to(tl.float32)[None, :] * inj[:, None]
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.store(out_ptr + row * stride_out + offs, out, mask=mask)
def _hc_combine(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor,
hc_count: int,
) -> torch.Tensor:
N, DIM = residual.shape
assert DIM % hc_count == 0
hc_dim = DIM // hc_count
assert block_output.shape == (N, hc_dim)
assert injection_logits.shape == (N, hc_count)
assert residual.stride(1) == 1
assert block_output.stride(1) == 1
assert injection_logits.stride(1) == 1
out = residual.new_empty(residual.shape)
BLOCK_SIZE = 512
_hc_combine_kernel[(N, triton.cdiv(hc_dim, BLOCK_SIZE))](
block_output,
residual,
injection_logits,
out,
block_output.stride(0),
residual.stride(0),
injection_logits.stride(0),
out.stride(0),
hc_dim,
hc_count,
BLOCK_SIZE,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return out
@triton.jit
def _hc_combine_norm_kernel(
block_ptr,
res_ptr,
inj_ptr,
w_ptr,
out_ptr,
y_ptr,
stride_block,
stride_res,
stride_inj,
stride_out,
stride_y,
HC_DIM: tl.constexpr,
HC: tl.constexpr,
W_SHARED: tl.constexpr,
EPS: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
) -> None:
HC_PAD: tl.constexpr = triton.next_power_of_2(HC)
NUM_TILES: tl.constexpr = triton.cdiv(HC_DIM, BLOCK_SIZE)
NUM_TILES_PAD: tl.constexpr = triton.next_power_of_2(NUM_TILES)
row = tl.program_id(0)
stream = tl.program_id(1)
offs_hc = tl.arange(0, HC_PAD)
mask_hc = offs_hc < HC
tile_ids = tl.arange(0, NUM_TILES_PAD)
offs_inner = tile_ids[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :]
mask_inner = offs_inner < HC_DIM
offs = stream * HC_DIM + offs_inner
# Shared norm weights repeat across streams; per-branch weights use the
# same flattened HC layout as the residual.
w_offs = offs_inner if W_SHARED else offs
if launch_pdl:
tl.extra.cuda.gdc_wait()
# Start the uncached residual load first, then issue the other combine
# loads before consuming any of them.
res = tl.load(res_ptr + row * stride_res + offs, mask_inner, other=0.0)
inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)
inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
inj = tl.sum(tl.where(offs_hc == stream, inj, 0.0))
# Round the materialized combine result before normalization. This matches
# the unfused combine -> RMSNorm boundary.
out = (res.to(tl.float32) + block.to(tl.float32) * inj).to(out_ptr.dtype.element_ty)
tl.store(out_ptr + row * stride_out + offs, out, mask=mask_inner)
out = out.to(tl.float32)
# Keep the two-axis reduction: flattening the padded tile is ~40% slower
# at decode sizes.
sum_sq = tl.sum(tl.sum(out * out, axis=1), axis=0)
rrms = tl.rsqrt(sum_sq / HC_DIM + EPS)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
# Loading the weight earlier helps decode but keeps the tile live across
# the reduction and regresses larger batches, so defer it to the norm.
w = tl.load(w_ptr + w_offs, mask_inner, other=0.0)
y = out * rrms
y += y * w.to(tl.float32)
tl.store(y_ptr + row * stride_y + offs, y, mask_inner)
def _hc_combine_norm(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor,
norm_weight: torch.Tensor,
eps: float,
hc_count: int,
) -> tuple[torch.Tensor, torch.Tensor]:
N, DIM = residual.shape
assert DIM % hc_count == 0
hc_dim = DIM // hc_count
assert block_output.shape == (N, hc_dim)
assert injection_logits.shape == (N, hc_count)
assert residual.stride(1) == 1
assert block_output.stride(1) == 1
assert injection_logits.stride(1) == 1
assert norm_weight.is_contiguous()
assert norm_weight.numel() in (hc_dim, DIM)
out = residual.new_empty(residual.shape)
y = residual.new_empty(residual.shape)
BLOCK_SIZE = 512
_hc_combine_norm_kernel[(N, hc_count)](
block_output,
residual,
injection_logits,
norm_weight,
out,
y,
block_output.stride(0),
residual.stride(0),
injection_logits.stride(0),
out.stride(0),
y.stride(0),
hc_dim,
hc_count,
W_SHARED=norm_weight.numel() == hc_dim,
EPS=eps,
BLOCK_SIZE=BLOCK_SIZE,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return out, y
def _same_shape_fake(x: torch.Tensor, *args) -> torch.Tensor:
return x.new_empty(x.shape)
def _hc_gate_mix_fake(
x: torch.Tensor, gate: torch.Tensor, hc_count: int
) -> torch.Tensor:
del gate
return x.new_empty((x.shape[0], x.shape[1] // hc_count))
def _hc_combine_fake(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor,
hc_count: int,
) -> torch.Tensor:
del block_output, injection_logits, hc_count
return residual.new_empty(residual.shape)
def _hc_combine_norm_fake(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor,
norm_weight: torch.Tensor,
eps: float,
hc_count: int,
) -> tuple[torch.Tensor, torch.Tensor]:
del block_output, injection_logits, norm_weight, eps, hc_count
return residual.new_empty(residual.shape), residual.new_empty(residual.shape)
direct_register_custom_op(
op_name="qwen4_exp_grouped_gemma_rmsnorm",
op_func=_grouped_gemma_rmsnorm,
fake_impl=_same_shape_fake,
)
direct_register_custom_op(
op_name="qwen4_exp_hc_silu",
op_func=_hc_silu,
fake_impl=_same_shape_fake,
)
direct_register_custom_op(
op_name="qwen4_exp_hc_gate_mix",
op_func=_hc_gate_mix,
fake_impl=_hc_gate_mix_fake,
)
direct_register_custom_op(
op_name="qwen4_exp_hc_combine",
op_func=_hc_combine,
fake_impl=_hc_combine_fake,
)
direct_register_custom_op(
op_name="qwen4_exp_hc_combine_norm",
op_func=_hc_combine_norm,
fake_impl=_hc_combine_norm_fake,
)
def grouped_gemma_rmsnorm(
x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
) -> torch.Tensor:
return torch.ops.vllm.qwen4_exp_grouped_gemma_rmsnorm(x, weight, eps, num_groups)
def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
return torch.ops.vllm.qwen4_exp_hc_silu(x, hc_count)
def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
return torch.ops.vllm.qwen4_exp_hc_gate_mix(x, gate, hc_count)
def hc_combine(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor,
hc_count: int,
) -> torch.Tensor:
return torch.ops.vllm.qwen4_exp_hc_combine(
residual, block_output, injection_logits, hc_count
)
def hc_combine_norm(
residual: torch.Tensor,
block_output: torch.Tensor,
injection_logits: torch.Tensor,
norm_weight: torch.Tensor,
eps: float,
hc_count: int,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.ops.vllm.qwen4_exp_hc_combine_norm(
residual,
block_output,
injection_logits,
norm_weight,
eps,
hc_count,
)
__all__ = [
"grouped_gemma_rmsnorm",
"hc_combine",
"hc_combine_norm",
"hc_gate_mix",
"hc_silu",
]
File diff suppressed because it is too large Load Diff
@@ -1,703 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused QSA prepare kernel for the AMD Qwen4Exp path.
Started as a copy of the NVIDIA kernel (``nvidia/ops/qsa_prepare.py``) and is
kept separate so either vendor can change its own copy without re-validating
the other. The state source select is two masked loads rather than a pointer
select, which ROCm Triton rejects.
"""
import torch
from torch import nn
from vllm.triton_utils import tl, triton
def supports_fused_pre_indexer(
rotary_emb: nn.Module,
head_dim: int,
num_kv_heads: int,
compress_ratio: int,
) -> bool:
"""Report whether this indexer's shapes match the fused kernel's assumptions.
The kernel hard-codes the rotary layout and the single-KV-head group
compression it was written for; everything it rejects has a working unfused
path.
"""
rotary_dim = int(rotary_emb.rotary_dim)
mrope_section = getattr(rotary_emb, "mrope_section", None)
return (
bool(getattr(rotary_emb, "is_neox_style", False))
and (
not mrope_section
or (
len(mrope_section) == 3
and sum(mrope_section) == rotary_dim // 2
and bool(getattr(rotary_emb, "mrope_interleaved", False))
)
)
and head_dim == 128
and rotary_dim == 64
and num_kv_heads == 1
and compress_ratio > 1
and compress_ratio & (compress_ratio - 1) == 0
)
@triton.jit
def _norm_rope(
x,
pos_t,
pos_h,
pos_w,
cos_sin_ptr,
cos_sin_stride,
norm_weight_ptr,
eps,
IS_MROPE: tl.constexpr,
MROPE_H: tl.constexpr,
MROPE_W: tl.constexpr,
):
"""Apply Gemma RMSNorm and selected-axis NeoX RoPE to register rows."""
TILE_T: tl.constexpr = x.shape[0]
TILE_H: tl.constexpr = x.shape[1]
D: tl.constexpr = x.shape[2]
ROWS: tl.constexpr = TILE_T * TILE_H
HALF: tl.constexpr = D // 2
QUARTER: tl.constexpr = D // 4
pairs = tl.arange(0, QUARTER)
if IS_MROPE:
# Qwen interleaves temporal, height, and width rotary pairs. Each axis
# still indexes the same position-major cos/sin table.
h_mask = ((pairs % 3) == 1) & (pairs <= 3 * MROPE_H)
w_mask = ((pairs % 3) == 2) & (pairs <= 3 * MROPE_W)
t_mask = ~(h_mask | w_mask)
base = cos_sin_ptr + pairs[None, :]
pos_rows = (pos_t, pos_h, pos_w)
axis_masks = (t_mask, h_mask, w_mask)
cos = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty)
sin = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty)
for axis in tl.static_range(3):
cos += tl.load(
base + pos_rows[axis][:, None] * cos_sin_stride,
mask=axis_masks[axis][None, :],
other=0,
)
sin += tl.load(
base + pos_rows[axis][:, None] * cos_sin_stride + QUARTER,
mask=axis_masks[axis][None, :],
other=0,
)
else:
cos = tl.load(cos_sin_ptr + pos_t[:, None] * cos_sin_stride + pairs[None, :])
sin = tl.load(
cos_sin_ptr + pos_t[:, None] * cos_sin_stride + QUARTER + pairs[None, :]
)
cos = tl.reshape(
tl.broadcast_to(cos[:, None, :], (TILE_T, TILE_H, QUARTER)),
(ROWS, QUARTER),
)
sin = tl.reshape(
tl.broadcast_to(sin[:, None, :], (TILE_T, TILE_H, QUARTER)),
(ROWS, QUARTER),
)
x = tl.reshape(x, (ROWS, D)).to(tl.float32)
weight = tl.load(norm_weight_ptr + tl.arange(0, D)).to(tl.float32) + 1.0
rrms = tl.rsqrt(tl.sum(x * x, axis=1) / D + eps)
y = (x * rrms[:, None] * weight[None, :]).to(cos.dtype)
rotated, passthrough = tl.split(
tl.permute(tl.reshape(y, (ROWS, 2, HALF)), (0, 2, 1))
)
r0, r1 = tl.split(tl.permute(tl.reshape(rotated, (ROWS, 2, QUARTER)), (0, 2, 1)))
out0 = r0 * cos - r1 * sin
out1 = r1 * cos + r0 * sin
rotated = tl.reshape(tl.permute(tl.join(out0, out1), (0, 2, 1)), (ROWS, HALF))
result = tl.reshape(tl.permute(tl.join(rotated, passthrough), (0, 2, 1)), (ROWS, D))
return tl.reshape(result, (TILE_T, TILE_H, D))
@triton.jit
def _to_dst_dtype(x, dst, scale):
"""Round to BF16 like the unfused path, then scale for an FP8 destination."""
out_ty = dst.dtype.element_ty
x = x.to(tl.bfloat16)
if out_ty == tl.float8e4nv:
x = x.to(tl.float32) / scale
return x.to(out_ty)
@triton.jit
def _store_rotated(dst, y, o1, o2, scale):
"""Store a normalized head whose first ``2 * len(o1)`` dims are rotated."""
HALF: tl.constexpr = o1.shape[0]
dims = tl.arange(0, y.shape[0])
rot = tl.arange(0, HALF)
tl.store(dst + dims, _to_dst_dtype(y, dst, scale), mask=dims >= 2 * HALF)
tl.store(dst + rot, _to_dst_dtype(o1, dst, scale))
tl.store(dst + HALF + rot, _to_dst_dtype(o2, dst, scale))
@triton.jit(
do_not_specialize=[
"num_tokens",
"num_state_blocks",
"num_compressed_blocks",
"num_k_work",
]
)
def _qsa_prepare_kernel(
q_ptr,
q_stride_token,
k_ptr,
k_stride_token,
pos_ptr,
pos_stride_axis,
pos_stride_token,
cos_sin_ptr,
q_norm_weight_ptr,
k_norm_weight_ptr,
eps,
q_out_ptr,
q_out_stride_token,
q_out_stride_head,
state_cache_ptr,
state_cache_stride_block,
state_cache_stride_token,
state_slots_ptr,
state_table_ptr,
state_table_stride_req,
query_start_loc_ptr,
logical_positions_ptr,
compressed_slots_ptr,
k_work_metadata_ptr,
compressed_cache_ptr,
compressed_cache_stride_block,
compressed_cache_stride_token,
num_tokens,
num_state_blocks,
num_compressed_blocks,
num_k_work,
HQ: tl.constexpr,
D: tl.constexpr,
TILE_T_Q: tl.constexpr,
TILE_H_Q: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
STATE_SIZE: tl.constexpr,
COMP_PAGE_SIZE: tl.constexpr,
IS_2D_POSITIONS: tl.constexpr,
IS_K_MROPE: tl.constexpr,
CACHE_HAS_ROPE_POS: tl.constexpr,
MROPE_H: tl.constexpr,
MROPE_W: tl.constexpr,
main_qkv_ptr,
main_q_norm_weight_ptr,
main_k_norm_weight_ptr,
main_eps,
main_q_out_ptr,
main_gate_out_ptr,
main_cache_ptr,
main_cache_stride_block,
main_cache_stride_token,
main_cache_stride_head,
main_slots_ptr,
main_k_scale,
main_v_scale,
MAIN_HQ: tl.constexpr,
MAIN_HK: tl.constexpr,
MAIN_D: tl.constexpr,
MAIN_PAGE_SIZE: tl.constexpr,
):
pid = tl.program_id(0)
num_index_work = num_k_work + tl.cdiv(num_tokens, TILE_T_Q) * tl.cdiv(HQ, TILE_H_Q)
if pid >= num_index_work:
# Main attention: one program per (token, Q or KV head) after the
# indexer work. RoPE covers the first D // 2 dims, the width of the
# shared cos/sin table.
main_pid = pid - num_index_work
token = (main_pid // (MAIN_HQ + MAIN_HK)).to(tl.int64)
head = main_pid % (MAIN_HQ + MAIN_HK)
HALF: tl.constexpr = D // 4
dims = tl.arange(0, MAIN_D)
rot = tl.arange(0, HALF)
row = main_qkv_ptr + token * (2 * (MAIN_HQ + MAIN_HK) * MAIN_D)
is_k = head >= MAIN_HQ
kv_head = head - MAIN_HQ
if is_k:
src = row + (2 * MAIN_HQ + kv_head) * MAIN_D
weight_ptr = main_k_norm_weight_ptr
else:
src = row + 2 * head * MAIN_D
weight_ptr = main_q_norm_weight_ptr
x = tl.load(src + dims).to(tl.float32)
inv_rms = tl.rsqrt(tl.sum(x * x, axis=0) / MAIN_D + main_eps)
w = tl.load(weight_ptr + dims).to(tl.float32) + 1.0
y = x * inv_rms * w
x1 = tl.load(src + rot).to(tl.float32)
x2 = tl.load(src + HALF + rot).to(tl.float32)
w1 = tl.load(weight_ptr + rot).to(tl.float32) + 1.0
w2 = tl.load(weight_ptr + HALF + rot).to(tl.float32) + 1.0
x1 = (x1 * inv_rms * w1).to(tl.bfloat16).to(tl.float32)
x2 = (x2 * inv_rms * w2).to(tl.bfloat16).to(tl.float32)
pos = tl.load(pos_ptr + token * pos_stride_token).to(tl.int64)
if IS_2D_POSITIONS:
pos_h = tl.load(pos_ptr + pos_stride_axis + token * pos_stride_token)
pos_w = tl.load(pos_ptr + 2 * pos_stride_axis + token * pos_stride_token)
is_h = (rot % 3 == 1) & (rot < 3 * MROPE_H)
is_w = (rot % 3 == 2) & (rot < 3 * MROPE_W)
pos = tl.where(
is_h, pos_h.to(tl.int64), tl.where(is_w, pos_w.to(tl.int64), pos)
)
cos = tl.load(cos_sin_ptr + pos * (D // 2) + rot).to(tl.float32)
sin = tl.load(cos_sin_ptr + pos * (D // 2) + HALF + rot).to(tl.float32)
o1 = x1 * cos - x2 * sin
o2 = x2 * cos + x1 * sin
if is_k:
slot = tl.load(main_slots_ptr + token).to(tl.int64)
if slot >= 0:
dst = (
main_cache_ptr
+ (slot // MAIN_PAGE_SIZE) * main_cache_stride_block
+ (slot % MAIN_PAGE_SIZE) * main_cache_stride_token
+ kv_head * main_cache_stride_head
)
_store_rotated(dst, y, o1, o2, main_k_scale)
v = tl.load(src + MAIN_HK * MAIN_D + dims)
tl.store(dst + MAIN_D + dims, _to_dst_dtype(v, dst, main_v_scale))
else:
out = (token * MAIN_HQ + head) * MAIN_D
_store_rotated(main_q_out_ptr + out, y, o1, o2, None)
gate = tl.load(src + MAIN_D + dims)
tl.store(main_gate_out_ptr + out + dims, gate)
return
# K work occupies the first programs; the remaining programs tile Q. This
# keeps both paths in one launch while leaving their register shapes
# independent.
if pid >= num_k_work:
q_pid = pid - num_k_work
num_head_tiles: tl.constexpr = tl.cdiv(HQ, TILE_H_Q)
token_tile = q_pid // num_head_tiles
head_tile = q_pid % num_head_tiles
tokens = (token_tile * TILE_T_Q + tl.arange(0, TILE_T_Q)).to(tl.int64)
heads = head_tile * TILE_H_Q + tl.arange(0, TILE_H_Q)
valid_tokens = tokens < num_tokens
valid_heads = heads < HQ
dims = tl.arange(0, D)
mask = valid_tokens[:, None, None] & valid_heads[None, :, None]
x = tl.load(
q_ptr
+ tokens[:, None, None] * q_stride_token
+ heads[None, :, None] * D
+ dims[None, None, :],
mask=mask,
other=0.0,
)
pos_t = tl.load(pos_ptr + tokens * pos_stride_token, mask=valid_tokens, other=0)
if IS_2D_POSITIONS:
pos_h = tl.load(
pos_ptr + pos_stride_axis + tokens * pos_stride_token,
mask=valid_tokens,
other=0,
)
pos_w = tl.load(
pos_ptr + 2 * pos_stride_axis + tokens * pos_stride_token,
mask=valid_tokens,
other=0,
)
else:
pos_h = pos_t
pos_w = pos_t
y = _norm_rope(
x,
pos_t,
pos_h,
pos_w,
cos_sin_ptr,
D // 2,
q_norm_weight_ptr,
eps,
IS_2D_POSITIONS,
MROPE_H,
MROPE_W,
)
tl.store(
q_out_ptr
+ tokens[:, None, None] * q_out_stride_token
+ heads[None, :, None] * q_out_stride_head
+ dims[None, None, :],
y,
mask=mask,
)
if pid < num_k_work:
# One work item owns one completed compression group. Work item zero
# additionally commits the request's current raw-K suffix below.
work_metadata = tl.load(k_work_metadata_ptr + pid * 2 + tl.arange(0, 2))
request, work_in_request = tl.split(work_metadata)
if request < 0:
return
query_start = tl.load(query_start_loc_ptr + request)
query_end = tl.load(query_start_loc_ptr + request + 1)
query_len = query_end - query_start
chunk_end = tl.load(logical_positions_ptr + query_end - 1)
chunk_start = chunk_end - query_len + 1
num_groups = (chunk_end + 1) // COMPRESS_RATIO - chunk_start // COMPRESS_RATIO
dims = tl.arange(0, D)
if work_in_request < num_groups:
first_boundary = (
(chunk_start + COMPRESS_RATIO) // COMPRESS_RATIO
) * COMPRESS_RATIO - 1
end_position = first_boundary + work_in_request * COMPRESS_RATIO
boundary_token = query_start + end_position - chunk_start
valid_token = (
(boundary_token >= query_start)
& (boundary_token < query_end)
& (boundary_token < num_tokens)
)
compressed_slot = tl.load(
compressed_slots_ptr + boundary_token,
mask=valid_token,
other=-1,
)
valid = (
valid_token
& (compressed_slot >= 0)
& (compressed_slot < num_compressed_blocks * COMP_PAGE_SIZE)
)
state_block = tl.load(state_table_ptr + request * state_table_stride_req)
state_block_valid = (state_block >= 0) & (state_block < num_state_blocks)
safe_state_block = tl.maximum(state_block, 0).to(tl.int64)
group_offsets = tl.arange(0, COMPRESS_RATIO)
source_positions = end_position - (COMPRESS_RATIO - 1) + group_offsets
source_in_chunk = source_positions >= chunk_start
source_in_cache = source_positions < chunk_start
source_tokens = query_start + source_positions - chunk_start
source_tokens_valid = (
(source_tokens >= query_start)
& (source_tokens < query_end)
& (source_tokens < num_tokens)
)
current_base = (
k_ptr
+ tl.maximum(source_tokens, 0).to(tl.int64)[:, None] * k_stride_token
)
cached_base = (
state_cache_ptr
+ safe_state_block * state_cache_stride_block
+ (source_positions % STATE_SIZE)[:, None] * state_cache_stride_token
)
# Only the first completed group can cross the chunk boundary, so
# every row comes from exactly one of the two sources. Selecting
# between the base pointers would need a pointer-typed tl.where,
# which ROCm's Triton rejects ("'tt.addptr' op failed to verify
# that result type matches ptr type"), so select the loaded values
# instead. The two masks are complementary, so no lane fetches from
# both and memory traffic is unchanged.
#
# Both bases still need the alignment stated explicitly. Without it
# Triton vectorizes these loads differently, which reorders the
# pooling reduction below; float addition is not associative, so the
# compressed rows stop matching the unfused path bit for bit and an
# FP8 cache rounds a near-zero pooled value to the wrong code.
current_base = tl.multiple_of(current_base, (8, 8))
cached_base = tl.multiple_of(cached_base, (8, 8))
from_chunk = valid & source_in_chunk[:, None] & source_tokens_valid[:, None]
from_cache = valid & source_in_cache[:, None] & state_block_valid
current = tl.load(current_base + dims[None, :], mask=from_chunk, other=0.0)
cached = tl.load(cached_base + dims[None, :], mask=from_cache, other=0.0)
source = tl.where(source_in_chunk[:, None], current, cached).to(tl.float32)
# Match the unfused path's BF16 pooled tensor before RMSNorm.
pooled = (
(tl.sum(source, axis=0) / COMPRESS_RATIO).to(tl.bfloat16).to(tl.float32)
)
first_position = end_position - (COMPRESS_RATIO - 1)
if CACHE_HAS_ROPE_POS:
# RoPE uses the first token in the pooled group. Its exact MRoPE
# coordinates may live in this chunk or the raw-state ring.
first_in_chunk = first_position >= chunk_start
first_token = query_start + first_position - chunk_start
first_token_valid = (
(first_token >= query_start)
& (first_token < query_end)
& (first_token < num_tokens)
)
safe_first_token = tl.maximum(first_token, 0)
load_current_position = first_in_chunk & first_token_valid
first_pos_t = tl.load(
pos_ptr + safe_first_token * pos_stride_token,
mask=load_current_position,
other=0,
)
if IS_2D_POSITIONS:
first_pos_h = tl.load(
pos_ptr + pos_stride_axis + safe_first_token * pos_stride_token,
mask=load_current_position,
other=0,
)
first_pos_w = tl.load(
pos_ptr
+ 2 * pos_stride_axis
+ safe_first_token * pos_stride_token,
mask=load_current_position,
other=0,
)
else:
first_pos_h = first_pos_t
first_pos_w = first_pos_t
tail = (
state_cache_ptr
+ safe_state_block * state_cache_stride_block
+ (first_position % STATE_SIZE) * state_cache_stride_token
+ D
).to(tl.pointer_type(tl.int64))
load_cached_position = ~first_in_chunk & state_block_valid
cached_pos_t = tl.load(tail, mask=load_cached_position, other=0)
cached_pos_h = tl.load(tail + 1, mask=load_cached_position, other=0)
cached_pos_w = tl.load(tail + 2, mask=load_cached_position, other=0)
pos_t = tl.where(first_in_chunk, first_pos_t.to(tl.int64), cached_pos_t)
pos_h = tl.where(first_in_chunk, first_pos_h.to(tl.int64), cached_pos_h)
pos_w = tl.where(first_in_chunk, first_pos_w.to(tl.int64), cached_pos_w)
else:
pos_t = first_position
pos_h = first_position
pos_w = first_position
y = _norm_rope(
tl.reshape(pooled, (1, 1, D)),
pos_t + tl.arange(0, 1),
pos_h + tl.arange(0, 1),
pos_w + tl.arange(0, 1),
cos_sin_ptr,
D // 2,
k_norm_weight_ptr,
eps,
IS_K_MROPE,
MROPE_H,
MROPE_W,
)
compressed_block = (compressed_slot // COMP_PAGE_SIZE).to(tl.int64)
compressed_row = compressed_slot % COMP_PAGE_SIZE
tl.store(
compressed_cache_ptr
+ compressed_block * compressed_cache_stride_block
+ compressed_row * compressed_cache_stride_token
+ dims,
tl.reshape(y, (D,)),
mask=valid,
)
if work_in_request == 0:
# This CTA may have just read historical rows from the circular buffer.
# Keep every lane past those loads before overwriting the same ring.
tl.debug_barrier()
# One CTA per request commits only the suffix retained by the ring.
num_state_rows = tl.minimum(query_len, STATE_SIZE)
for state_offset in tl.range(0, num_state_rows):
token = (query_end - num_state_rows + state_offset).to(tl.int64)
valid_token = (
(token >= query_start) & (token < query_end) & (token < num_tokens)
)
slot = tl.load(state_slots_ptr + token, mask=valid_token, other=-1)
valid_slot = (
valid_token & (slot >= 0) & (slot < num_state_blocks * STATE_SIZE)
)
safe_slot = tl.maximum(slot, 0)
state_row = (
state_cache_ptr
+ (safe_slot // STATE_SIZE).to(tl.int64) * state_cache_stride_block
+ (safe_slot % STATE_SIZE) * state_cache_stride_token
)
k = tl.load(
k_ptr + tl.maximum(token, 0) * k_stride_token + dims,
mask=valid_slot,
other=0.0,
)
tl.store(state_row + dims, k, mask=valid_slot)
if CACHE_HAS_ROPE_POS:
pos_t = tl.load(
pos_ptr + tl.maximum(token, 0) * pos_stride_token,
mask=valid_slot,
other=0,
)
if IS_2D_POSITIONS:
pos_h = tl.load(
pos_ptr
+ pos_stride_axis
+ tl.maximum(token, 0) * pos_stride_token,
mask=valid_slot,
other=0,
)
pos_w = tl.load(
pos_ptr
+ 2 * pos_stride_axis
+ tl.maximum(token, 0) * pos_stride_token,
mask=valid_slot,
other=0,
)
else:
pos_h = pos_t
pos_w = pos_t
tail = (state_row + D).to(tl.pointer_type(tl.int64))
tl.store(tail, pos_t.to(tl.int64), mask=valid_slot)
tl.store(tail + 1, pos_h.to(tl.int64), mask=valid_slot)
tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot)
def qsa_prepare(
q: torch.Tensor,
k: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
q_norm_weight: torch.Tensor,
k_norm_weight: torch.Tensor,
eps: float,
q_out: torch.Tensor,
state_cache: torch.Tensor,
state_slots: torch.Tensor,
state_block_table: torch.Tensor,
query_start_loc: torch.Tensor,
logical_positions: torch.Tensor,
compressed_cache: torch.Tensor,
compressed_slots: torch.Tensor,
k_work_metadata: torch.Tensor,
*,
compress_ratio: int,
mrope_section: tuple[int, int, int] | None,
rope_pos_offset: int | None,
main_qkv: torch.Tensor,
main_q_norm_weight: torch.Tensor,
main_k_norm_weight: torch.Tensor,
main_eps: float,
main_kv_cache: torch.Tensor,
main_slot_mapping: torch.Tensor,
main_k_scale: float,
main_v_scale: float,
main_gate_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Normalize Q, compress K, then update the circular raw state.
Also prepares the main attention (QK-norm/RoPE, gate copy, K/V cache write)
and returns its Q and gate. ``main_gate_out`` lets the gate land in a
caller-owned buffer, e.g. one that outlives an opaque custom op.
"""
num_tokens = q.shape[0]
main_head_dim = main_kv_cache.shape[-1] // 2
num_main_kv_heads = main_kv_cache.shape[2]
num_main_q_heads = main_qkv.shape[1] // (2 * main_head_dim) - num_main_kv_heads
main_q_out = main_qkv.new_empty(num_tokens, num_main_q_heads, main_head_dim)
if main_gate_out is None:
main_gate_out = torch.empty_like(main_q_out)
else:
assert main_gate_out.shape == main_q_out.shape
assert main_gate_out.is_contiguous()
if num_tokens == 0:
return main_q_out, main_gate_out
num_q_heads, head_dim = q_out.shape[1:]
assert cos_sin_cache.shape[-1] * 2 == head_dim
assert q.shape == (num_tokens, num_q_heads * head_dim)
assert k.shape == (num_tokens, head_dim)
assert q.stride(-1) == 1
assert k.stride(-1) == 1
assert q_out.stride(-1) == 1
assert cos_sin_cache.is_contiguous()
assert state_cache.stride(-1) == 1
assert compressed_cache.stride(-1) == 1
assert k_work_metadata.ndim == 2 and k_work_metadata.shape[1] == 2
is_2d_positions = positions.ndim == 2
is_k_mrope = bool(mrope_section)
cache_has_rope_pos = rope_pos_offset is not None
assert rope_pos_offset is None or rope_pos_offset == head_dim
if is_2d_positions:
assert positions.shape == (3, num_tokens)
assert is_k_mrope
pos_stride_axis, pos_stride_token = positions.stride()
else:
assert positions.shape == (num_tokens,)
pos_stride_axis, pos_stride_token = 0, positions.stride(0)
section = mrope_section if mrope_section is not None else (0, 0, 0)
assert len(section) == 3
qkv_width = 2 * (num_main_q_heads + num_main_kv_heads) * main_head_dim
assert main_qkv.shape == (num_tokens, qkv_width) and main_qkv.is_contiguous()
assert main_slot_mapping.shape == (num_tokens,)
# Swept on MI355X (64-lane wavefronts) over TILE_T_Q 1-16, TILE_H_Q 1-4 and
# num_warps 1/2/4, from 4 to 16384 tokens: no config beat these by more
# than noise, and 2 or 4 warps were slower everywhere.
if num_tokens <= 4096:
TILE_T_Q, TILE_H_Q = 2, 2
else:
TILE_T_Q, TILE_H_Q = 2, 4
num_k_work = k_work_metadata.shape[0]
num_q_work = triton.cdiv(num_tokens, TILE_T_Q) * triton.cdiv(num_q_heads, TILE_H_Q)
num_main_work = num_tokens * (num_main_q_heads + num_main_kv_heads)
_qsa_prepare_kernel[(num_k_work + num_q_work + num_main_work,)](
q,
q.stride(0),
k,
k.stride(0),
positions,
pos_stride_axis,
pos_stride_token,
cos_sin_cache,
q_norm_weight,
k_norm_weight,
eps,
q_out,
q_out.stride(0),
q_out.stride(1),
state_cache,
state_cache.stride(0),
state_cache.stride(1),
state_slots,
state_block_table,
state_block_table.stride(0),
query_start_loc,
logical_positions,
compressed_slots,
k_work_metadata,
compressed_cache,
compressed_cache.stride(0),
compressed_cache.stride(1),
num_tokens,
state_cache.shape[0],
compressed_cache.shape[0],
num_k_work,
HQ=num_q_heads,
D=head_dim,
TILE_T_Q=TILE_T_Q,
TILE_H_Q=TILE_H_Q,
COMPRESS_RATIO=compress_ratio,
STATE_SIZE=state_cache.shape[1],
COMP_PAGE_SIZE=compressed_cache.shape[1],
IS_2D_POSITIONS=is_2d_positions,
IS_K_MROPE=is_k_mrope,
CACHE_HAS_ROPE_POS=cache_has_rope_pos,
MROPE_H=section[1],
MROPE_W=section[2],
main_qkv_ptr=main_qkv,
main_q_norm_weight_ptr=main_q_norm_weight,
main_k_norm_weight_ptr=main_k_norm_weight,
main_eps=main_eps,
main_q_out_ptr=main_q_out,
main_gate_out_ptr=main_gate_out,
main_cache_ptr=main_kv_cache,
main_cache_stride_block=main_kv_cache.stride(0),
main_cache_stride_token=main_kv_cache.stride(1),
main_cache_stride_head=main_kv_cache.stride(2),
main_slots_ptr=main_slot_mapping,
main_k_scale=main_k_scale,
main_v_scale=main_v_scale,
MAIN_HQ=num_main_q_heads,
MAIN_HK=num_main_kv_heads,
MAIN_D=main_head_dim,
MAIN_PAGE_SIZE=main_kv_cache.shape[1],
num_warps=1,
)
return main_q_out, main_gate_out
__all__ = ["qsa_prepare", "supports_fused_pre_indexer"]
@@ -1,47 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.models.common.ops import sequence_parallel as common_sp
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import direct_register_custom_op
def _sp_shard_op(x: torch.Tensor) -> torch.Tensor:
output = common_sp.sp_shard(x)
# A functional custom op must not return a view of its input.
unpadded = x.shape[0] % get_tensor_model_parallel_world_size() == 0
return output.clone() if unpadded else output
def _sp_local_fake(x: torch.Tensor) -> torch.Tensor:
tp_size = get_tensor_model_parallel_world_size()
return x.new_empty((cdiv(x.shape[0], tp_size), *x.shape[1:]))
# Dynamo would specialize the padding branch on the traced token count, and
# vLLM drops the resulting guards. Keep the branch opaque when compiling.
direct_register_custom_op(
op_name="sp_reduce_scatter",
op_func=common_sp.sp_reduce_scatter,
fake_impl=_sp_local_fake,
)
direct_register_custom_op(
op_name="sp_shard",
op_func=_sp_shard_op,
fake_impl=_sp_local_fake,
)
def sp_reduce_scatter(x: torch.Tensor) -> torch.Tensor:
if torch.compiler.is_compiling():
return torch.ops.vllm.sp_reduce_scatter(x)
return common_sp.sp_reduce_scatter(x)
def sp_shard(x: torch.Tensor) -> torch.Tensor:
if torch.compiler.is_compiling():
return torch.ops.vllm.sp_shard(x)
return common_sp.sp_shard(x)
-883
View File
@@ -1,883 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GPU-resident Qwen4Exp position-learning enhancement layers."""
from collections.abc import Iterable, Sequence
import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.distributed import get_etp_group, get_tp_group
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
from vllm.model_executor.layers.mamba.abstract import MambaBase
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateDtypeCalculator,
MambaStateShapeCalculator,
is_conv_state_dim_first,
)
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_reduce_scatter,
sp_shard,
)
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.short_conv_attn import (
PleShortConvAttentionBackend,
PleShortConvAttentionMetadata,
)
from ..common.ngram_embedding import (
Qwen4ExpPLEDeviceEmbedding,
Qwen4ExpPLEEmbeddingMethod,
Qwen4ExpPLENvFp4EmbeddingMethod,
Qwen4ExpPLEPinnedHostEmbedding,
)
from ..common.ops.ple import ple_conv, ple_gate, ple_ngram_ids
logger = init_logger(__name__)
class Qwen4ExpPLEGroupedNorm(nn.Module):
def __init__(
self,
hidden_size: int,
eps: float,
group_size: int | None,
dtype: torch.dtype | None,
) -> None:
super().__init__()
if group_size is not None and hidden_size % group_size:
raise ValueError(
f"hidden_size ({hidden_size}) must be divisible by "
f"group_size ({group_size})"
)
self.eps = eps
self.group_size = group_size
self.weight = nn.Parameter(torch.zeros(hidden_size, dtype=dtype))
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
input_dtype = hidden_states.dtype
hidden_states = hidden_states.float()
if self.group_size is None:
variance = hidden_states.square().mean(dim=-1, keepdim=True)
normalized = hidden_states * torch.rsqrt(variance + self.eps)
else:
grouped = hidden_states.unflatten(
-1, (hidden_states.shape[-1] // self.group_size, self.group_size)
)
variance = grouped.square().mean(dim=-1, keepdim=True)
normalized = (grouped * torch.rsqrt(variance + self.eps)).flatten(-2)
return (normalized * (1.0 + self.weight.float())).to(input_dtype)
class Qwen4ExpNGramEmbedding(nn.Module):
_MASK64 = (1 << 64) - 1
_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
_SPLITMIX_M1 = 0xBF58476D1CE4E5B9
_SPLITMIX_M2 = 0x94D049BB133111EB
_PLE_LAYER_PRIME = 10007
@classmethod
def _splitmix64(cls, value: int) -> int:
"""Mix an integer into a deterministic unsigned 64-bit value."""
value = (value + cls._SPLITMIX_GAMMA) & cls._MASK64
value = ((value ^ (value >> 30)) * cls._SPLITMIX_M1) & cls._MASK64
value = ((value ^ (value >> 27)) * cls._SPLITMIX_M2) & cls._MASK64
return (value ^ (value >> 31)) & cls._MASK64
@staticmethod
def _is_prime_64(value: int) -> bool:
"""Return whether a 64-bit integer is prime."""
if value < 2:
return False
for prime in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37):
if value % prime == 0:
return value == prime
exponent = value - 1
shifts = 0
while exponent % 2 == 0:
exponent //= 2
shifts += 1
for base in (2, 325, 9375, 28178, 450775, 9780504, 1795265022):
if base % value == 0:
continue
witness = pow(base, exponent, value)
if witness in (1, value - 1):
continue
for _ in range(shifts - 1):
witness = pow(witness, 2, value)
if witness == value - 1:
break
else:
return False
return True
@classmethod
def _nth_prime_after(cls, start: int, count: int) -> int:
"""Return the ``count``-th prime strictly greater than ``start``."""
prime = int(start)
for _ in range(count):
candidate = prime + 1
if candidate <= 2:
prime = 2
continue
if candidate % 2 == 0:
candidate += 1
while not cls._is_prime_64(candidate):
candidate += 2
prime = candidate
return prime
@classmethod
def _make_layer_multipliers(
cls,
*,
ngram_size: int,
unigram_vocab_size: int,
seed: int,
ple_dense_layer_id: int,
) -> list[int]:
"""Build deterministic hash multipliers for one PLE layer."""
max_multiplier = ((1 << 63) - 1) // unigram_vocab_size
half_bound = max(1, max_multiplier // 2)
base_seed = seed + cls._PLE_LAYER_PRIME * ple_dense_layer_id
multipliers = []
for index in range(ngram_size):
value = base_seed + cls._SPLITMIX_GAMMA * (index + 1)
multipliers.append(2 * (cls._splitmix64(value) % half_bound) + 1)
return multipliers
@classmethod
def _make_vocab_layout(
cls,
*,
ngram_vocab_size_base: int,
ngram_heads: int,
ple_dense_layer_id: int,
) -> tuple[list[int], list[int], int]:
"""Build per-head vocabulary sizes, offsets, and total row count."""
sizes: list[int] = []
offsets: list[int] = []
offset = 0
for local_head in range(ngram_heads):
global_head = ple_dense_layer_id * ngram_heads + local_head
size = cls._nth_prime_after(ngram_vocab_size_base - 1, global_head + 1)
sizes.append(size)
offsets.append(offset)
offset += size
return sizes, offsets, offset
def __init__(
self,
config: Qwen4ExpTextConfig,
embedding_dim: int,
ple_dense_layer_id: int,
max_total_tokens: int,
prefix: str,
layer_name: str,
*,
data_parallel_rank: int = 0,
quant_config: QuantizationConfig | None = None,
params_dtype: torch.dtype | None = None,
use_sequence_parallel: bool = False,
) -> None:
super().__init__()
self.use_sequence_parallel = use_sequence_parallel
self.embedding_dim = embedding_dim
self.layer_name = layer_name
self.ngram_size = int(config.ngram_size)
self.heads_per_ngram = int(config.heads_per_ngram)
self.ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram
if self.ngram_size < 2:
raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}")
if self.heads_per_ngram <= 0:
raise ValueError(f"heads_per_ngram must be > 0, got {self.heads_per_ngram}")
if embedding_dim % self.ngram_heads:
raise ValueError(
"ple_embed_dim must be divisible by total ngram heads: "
f"{embedding_dim} % {self.ngram_heads} != 0"
)
self.head_dim = embedding_dim // self.ngram_heads
self.eos_token_id = int(config.eos_token_id)
self.unigram_vocab_size = int(config.vocab_size)
self.split_ngram_parts = int(getattr(config, "split_ngram_parts", 512))
if self.split_ngram_parts <= 0:
raise ValueError("split_ngram_parts must be positive")
multipliers = self._make_layer_multipliers(
ngram_size=self.ngram_size,
unigram_vocab_size=self.unigram_vocab_size,
seed=int(getattr(config, "seed", 1234)),
ple_dense_layer_id=ple_dense_layer_id,
)
self.register_buffer(
"layer_multipliers",
torch.tensor(multipliers, dtype=torch.long),
persistent=True,
)
sizes, offsets, total_vocab_size = self._make_vocab_layout(
ngram_vocab_size_base=int(config.ngram_vocab_size_base),
ngram_heads=self.ngram_heads,
ple_dense_layer_id=ple_dense_layer_id,
)
self.register_buffer(
"ngram_heads_vocab_sizes",
torch.tensor(sizes, dtype=torch.long),
persistent=True,
)
self.register_buffer(
"ngram_heads_offsets",
torch.tensor(offsets, dtype=torch.long),
persistent=True,
)
divisor = int(config.make_ngram_vocab_size_divisible_by)
padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor
if params_dtype is None:
params_dtype = torch.get_default_dtype()
embedding_prefix = f"{prefix}.ngram_embedding"
embedding_quant_method = Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
embedding_prefix,
getattr(config, "ple_embedding_dtype", None),
)
if isinstance(embedding_quant_method, Qwen4ExpPLENvFp4EmbeddingMethod):
raise NotImplementedError(
"NVFP4 Qwen4Exp PLE tables are not supported on ROCm yet; use a "
"checkpoint with a BF16 or FP8 PLE table."
)
engram_config = get_current_vllm_config().engram_config
embedding_cls = (
Qwen4ExpPLEPinnedHostEmbedding
if engram_config is not None and engram_config.cpu_offload
else Qwen4ExpPLEDeviceEmbedding
)
# A TP reduce-scatter covers all vocabulary owners only when ETP=TP.
self.use_reduce_scatter = (
self.use_sequence_parallel
and get_etp_group().world_size == get_tp_group().world_size
)
self.ngram_embedding = embedding_cls(
padded_vocab_size,
self.head_dim,
params_dtype=params_dtype,
padding_size=divisor,
prefix=embedding_prefix,
embedding_method=embedding_quant_method,
num_ngram_heads=self.ngram_heads,
max_total_tokens=max_total_tokens,
data_parallel_rank=data_parallel_rank,
reduce_results=not self.use_reduce_scatter,
)
logger.info(
"Initialized AMD PLE embedding %s: quantization_method=%s, "
"weight_dtype=%s, weight_device=%s, pinned=%s",
embedding_prefix,
type(embedding_quant_method).__name__,
self.ngram_embedding.weight.dtype,
self.ngram_embedding.weight.device,
self.ngram_embedding.weight.is_pinned(),
)
def _sp_shard_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor:
"""Shard embedding token rows, optionally reducing vocabulary contributions."""
# Each embedding element has one vocabulary owner, including FP8 bytes.
output_dtype = embeddings.dtype
if is_fp8(embeddings):
embeddings = embeddings.view(torch.int8)
if self.use_reduce_scatter:
embeddings = sp_reduce_scatter(embeddings)
else:
embeddings = sp_shard(embeddings)
return embeddings.view(output_dtype)
def forward(
self,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> torch.Tensor:
"""Return full embeddings or a token shard after combining vocabulary owners."""
input_ids = input_ids.reshape(-1)
ngram_ids = input_ids.new_empty(
(input_ids.shape[0], self.ngram_heads), dtype=torch.int64
)
torch.ops.vllm.qwen4_exp_amd_ple_ngram_ids(
input_ids,
query_start_loc,
ngram_context,
ngram_ids,
self.layer_name,
)
embedding = self.ngram_embedding
if embedding.supports_prefetch:
output = ngram_ids.new_empty(
(ngram_ids.shape[0], self.embedding_dim),
dtype=embedding.weight.dtype,
)
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned(
ngram_ids,
output,
self.layer_name,
)
else:
output = ngram_ids.new_empty(
(ngram_ids.shape[0], self.embedding_dim),
dtype=embedding.params_dtype,
)
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding(
ngram_ids,
output,
self.layer_name,
)
if not self.use_sequence_parallel:
return output
return self._sp_shard_embeddings(output)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load hash buffers and checkpoint-split embedding rows."""
persistent_buffers = {
"layer_multipliers": self.layer_multipliers,
"ngram_heads_offsets": self.ngram_heads_offsets,
"ngram_heads_vocab_sizes": self.ngram_heads_vocab_sizes,
}
loaded: set[str] = set()
regular_weights: list[tuple[str, torch.Tensor]] = []
shard_prefix = "ngram_embedding.shard_"
for name, loaded_weight in weights:
leaf_name = name.rsplit(".", 1)[-1]
if leaf_name.startswith("hashstats_") or leaf_name == "token_lookup":
continue
if name in persistent_buffers:
buffer = persistent_buffers[name]
if buffer.shape != loaded_weight.shape:
raise ValueError(
f"Shape mismatch for {name}: expected "
f"{tuple(buffer.shape)}, got {tuple(loaded_weight.shape)}"
)
buffer.copy_(loaded_weight.to(device=buffer.device, dtype=buffer.dtype))
loaded.add(name)
continue
if name.startswith(shard_prefix) and name.endswith(".weight"):
shard_text = name[len(shard_prefix) : -len(".weight")]
if not shard_text.isdigit():
regular_weights.append((name, loaded_weight))
continue
shard_index = int(shard_text)
if shard_index >= self.split_ngram_parts:
raise ValueError(
f"PLE embedding shard index {shard_index} exceeds "
f"split_ngram_parts={self.split_ngram_parts}"
)
embedding = self.ngram_embedding
shard_size = (
embedding.org_vocab_size + self.split_ngram_parts - 1
) // self.split_ngram_parts
checkpoint_start = shard_index * shard_size
expected_rows = max(
0,
min(shard_size, embedding.org_vocab_size - checkpoint_start),
)
expected_shape = (expected_rows, embedding.embedding_dim)
if tuple(loaded_weight.shape) != expected_shape:
raise ValueError(
f"Shape mismatch for PLE embedding shard {shard_index}: "
f"expected {expected_shape}, got "
f"{tuple(loaded_weight.shape)}"
)
embedding.weight.weight_loader(
embedding.weight,
loaded_weight,
checkpoint_start=checkpoint_start,
)
loaded.add("ngram_embedding.weight")
continue
regular_weights.append((name, loaded_weight))
if regular_weights:
loaded.update(AutoWeightsLoader(self).load_weights(regular_weights))
return loaded
class Qwen4ExpPLELayer(nn.Module, MambaBase):
def __init__(
self,
config: Qwen4ExpTextConfig,
vllm_config: VllmConfig,
layer_idx: int = 0,
ple_dense_layer_id: int | None = None,
prefix: str = "",
use_sequence_parallel: bool = False,
) -> None:
super().__init__()
self.use_sequence_parallel = use_sequence_parallel
model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.model_config: ModelConfig = model_config
self.cache_config: CacheConfig = cache_config
self.layer_idx = layer_idx
self.ple_dense_layer_id = (
int(ple_dense_layer_id)
if ple_dense_layer_id is not None
else int(layer_idx)
)
self.prefix = prefix
self.hidden_size = int(config.hidden_size)
self.hc_count = config.hc_count
self.hc_hidden_size = self.hidden_size * self.hc_count
self.conv_kernel_size = int(config.ple_conv_kernel_size)
self.short_conv_dilation = int(config.ngram_size)
self.conv_state_len = (self.conv_kernel_size - 1) * self.short_conv_dilation
self.num_spec_tokens = vllm_config.num_speculative_tokens
self.activation = "silu"
self.ple_embedding: nn.Module = Qwen4ExpNGramEmbedding(
config,
int(config.ple_embed_dim),
self.ple_dense_layer_id,
vllm_config.scheduler_config.max_num_batched_tokens,
f"{prefix}.ple_embedding",
prefix,
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
quant_config=quant_config,
params_dtype=model_config.dtype,
use_sequence_parallel=self.use_sequence_parallel,
)
# Projection weights stay replicated even when their token rows are sharded.
self.kv_proj = MergedColumnParallelLinear(
int(config.ple_embed_dim),
[self.hc_hidden_size, self.hidden_size],
bias=False,
params_dtype=model_config.dtype,
quant_config=quant_config,
prefix=f"{prefix}.kv_proj",
disable_tp=True,
)
norm_args = (
self.hc_hidden_size,
config.rms_norm_eps,
self.hidden_size,
model_config.dtype,
)
self.norm_key = Qwen4ExpPLEGroupedNorm(*norm_args)
self.norm_query = Qwen4ExpPLEGroupedNorm(*norm_args)
self.norm_conv = Qwen4ExpPLEGroupedNorm(*norm_args)
self.conv1d = nn.Conv1d(
self.hc_hidden_size,
self.hc_hidden_size,
self.conv_kernel_size,
groups=self.hc_hidden_size,
padding=self.conv_state_len,
dilation=self.short_conv_dilation,
bias=False,
dtype=model_config.dtype,
)
nn.init.zeros_(self.conv1d.weight)
self.conv1d.weight._no_reinit = True
self.kv_cache = (torch.tensor([]),)
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
@property
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.SHORT_CONV
@property
def is_kv_cache_tp_replicated(self) -> bool:
return True
def get_attn_backend(self) -> type[PleShortConvAttentionBackend]:
return PleShortConvAttentionBackend
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
return MambaStateDtypeCalculator.short_conv_state_dtype(
self.model_config.dtype, self.cache_config.mamba_cache_dtype
)
def get_state_shape(self) -> Sequence[tuple[int, ...]]:
return MambaStateShapeCalculator.short_conv_state_shape(
tp_world_size=1,
intermediate_size=self.hc_hidden_size,
conv_kernel=self.conv_state_len + 1,
num_spec=self.num_spec_tokens,
)
def _short_conv_dilated_dispatch(
self,
inputs: torch.Tensor,
residual: torch.Tensor,
outer_residual: torch.Tensor | None,
metadata: PleShortConvAttentionMetadata,
conv_state: torch.Tensor,
conv_weights: torch.Tensor,
) -> None:
num_prefills = metadata.num_prefills
num_decodes = metadata.num_decodes
num_decode_tokens = metadata.num_decode_tokens
num_prefill_tokens = metadata.num_prefill_tokens
has_prefill = num_prefills > 0
has_decode = num_decodes > 0
has_spec = metadata.spec_sequence_masks is not None
has_non_spec = has_prefill or has_decode
inputs = inputs[: metadata.num_actual_tokens]
residual = residual[: metadata.num_actual_tokens]
if outer_residual is not None:
outer_residual = outer_residual[: metadata.num_actual_tokens]
spec_token_indices = None
non_spec_token_indices = None
if has_spec and has_non_spec:
assert metadata.spec_token_indx is not None
assert metadata.non_spec_token_indx is not None
spec_token_indices = metadata.spec_token_indx
non_spec_token_indices = metadata.non_spec_token_indx
if has_spec:
assert metadata.spec_state_indices_tensor is not None
query_start_loc = metadata.spec_query_start_loc
num_accepted_tokens = metadata.num_accepted_tokens
assert query_start_loc is not None
assert num_accepted_tokens is not None
spec_state_indices = metadata.spec_state_indices_tensor[
: metadata.num_spec_decodes
]
# Mixed batches stay in their original row order; the kernels map
# logical spec/non-spec rows instead of materializing both groups.
ple_conv(
inputs=inputs,
residual=residual,
conv_state=conv_state,
conv_weights=conv_weights,
state_indices=spec_state_indices,
outer_residual=outer_residual,
mode="spec",
dilation=self.short_conv_dilation,
query_start_loc=query_start_loc,
num_accepted_tokens=num_accepted_tokens,
spec_query_len=metadata.spec_query_len,
token_indices=spec_token_indices,
)
if not has_non_spec:
return
state_indices = metadata.state_indices_tensor
assert state_indices is not None
if has_prefill:
state_indices_d, state_indices_p = torch.split(
state_indices, [num_decodes, num_prefills], dim=0
)
if non_spec_token_indices is None:
inputs_d, inputs_p = torch.split(
inputs, [num_decode_tokens, num_prefill_tokens], dim=0
)
residual_d, residual_p = torch.split(
residual, [num_decode_tokens, num_prefill_tokens], dim=0
)
outer_residual_d: torch.Tensor | None = None
outer_residual_p: torch.Tensor | None = None
if outer_residual is not None:
outer_residual_d, outer_residual_p = torch.split(
outer_residual,
[num_decode_tokens, num_prefill_tokens],
dim=0,
)
token_indices_d = None
token_indices_p = None
else:
inputs_d = inputs_p = inputs
residual_d = residual_p = residual
outer_residual_d = outer_residual_p = outer_residual
token_indices_d, token_indices_p = torch.split(
non_spec_token_indices,
[num_decode_tokens, num_prefill_tokens],
dim=0,
)
if has_decode:
ple_conv(
inputs=inputs_d,
residual=residual_d,
conv_state=conv_state,
conv_weights=conv_weights,
state_indices=state_indices_d,
outer_residual=outer_residual_d,
mode="decode",
dilation=self.short_conv_dilation,
has_initial_states=metadata.has_initial_states_d,
token_indices=token_indices_d,
)
query_start_loc = metadata.query_start_loc_p
if query_start_loc is None:
raise ValueError("query_start_loc is required for prefill short-conv")
has_initial_states = metadata.has_initial_states_p
if has_initial_states is None:
raise ValueError(
"has_initial_states_p is required for prefill short-conv"
)
ple_conv(
inputs=inputs_p,
residual=residual_p,
conv_state=conv_state,
conv_weights=conv_weights,
state_indices=state_indices_p,
outer_residual=outer_residual_p,
mode="prefill",
dilation=self.short_conv_dilation,
query_start_loc=query_start_loc,
has_initial_states=has_initial_states,
token_indices=token_indices_p,
)
else:
num_decode_rows = (
non_spec_token_indices.numel()
if non_spec_token_indices is not None
else inputs.size(0)
)
ple_conv(
inputs=inputs,
residual=residual,
conv_state=conv_state,
conv_weights=conv_weights,
state_indices=state_indices[:num_decode_rows],
outer_residual=outer_residual,
mode="decode",
dilation=self.short_conv_dilation,
has_initial_states=metadata.has_initial_states_d,
token_indices=non_spec_token_indices,
)
def _short_conv(
self,
inputs: torch.Tensor,
residual: torch.Tensor,
outer_residual: torch.Tensor | None,
) -> None:
"""Update convolution state, optionally fusing both residual additions."""
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
# Profiling omits all metadata or this Mamba entry. Short convolution
# is a no-op there, but preserve the outer residual addition.
if attn_metadata is None:
if outer_residual is not None:
residual.add_(outer_residual)
return
if not isinstance(attn_metadata, dict):
raise RuntimeError(
"PLE short-conv expects per-layer attention metadata dict "
f"during inference, got {type(attn_metadata).__name__}."
)
layer_attn_metadata = attn_metadata.get(self.prefix)
if layer_attn_metadata is None:
if outer_residual is not None:
residual.add_(outer_residual)
return
if not isinstance(layer_attn_metadata, PleShortConvAttentionMetadata):
raise TypeError(
"Expected PleShortConvAttentionMetadata for layer "
f"'{self.prefix}', got "
f"{type(layer_attn_metadata).__name__}."
)
conv_state = self.kv_cache[0]
# Canonicalize both backend cache layouts to [slot, channel, window].
if not is_conv_state_dim_first():
conv_state = conv_state.transpose(-1, -2)
conv_weights = self.conv1d.weight.squeeze(1)
state_capacity = self.conv_state_len + self.num_spec_tokens
if state_capacity > 0:
if conv_state.size(-1) < state_capacity:
raise RuntimeError(
"PLE short-conv cache is smaller than expected for "
f"dilated convolution: got {conv_state.size(-1)}, "
f"expect at least {state_capacity}."
)
conv_state = conv_state[..., -state_capacity:]
self._short_conv_dilated_dispatch(
inputs=inputs,
residual=residual,
outer_residual=outer_residual,
metadata=layer_attn_metadata,
conv_state=conv_state,
conv_weights=conv_weights.to(dtype=inputs.dtype),
)
def forward(
self,
hidden_states: torch.Tensor,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> torch.Tensor:
input_ids = input_ids.reshape(-1)
if (
not self.use_sequence_parallel
and input_ids.shape[0] != hidden_states.shape[0]
):
raise ValueError(
"PLE expects input_ids and hidden_states to have the same "
f"token length, got {input_ids.shape[0]} and "
f"{hidden_states.shape[0]}"
)
embeddings = self.ple_embedding(input_ids, query_start_loc, ngram_context)
embeddings = self.ple_embedding.ngram_embedding.dequantize(
embeddings, hidden_states.dtype
)
kv, _ = self.kv_proj(embeddings)
key, value = kv.split(self.kv_proj.output_sizes, dim=-1)
gated_output, conv_input = ple_gate(
key,
value,
hidden_states,
self.norm_key.weight,
self.norm_query.weight,
self.norm_conv.weight,
self.norm_key.eps,
)
# The short-conv op is a piecewise-graph splitting op: state routing
# reads runtime request metadata. It accumulates the convolution and
# the outer residual into gated_output.
if self.use_sequence_parallel:
# Only convolution needs full tokens. Add residuals on local shards.
conv_input = sp_all_gather(conv_input)[: input_ids.shape[0]]
# Metadata may omit padding tokens or all tokens during profiling.
conv_output = torch.zeros_like(conv_input)
torch.ops.vllm.qwen4_exp_ple_short_conv(
conv_input, conv_output, None, self.prefix
)
gated_output.add_(sp_shard(conv_output))
gated_output.add_(hidden_states)
else:
torch.ops.vllm.qwen4_exp_ple_short_conv(
conv_input,
gated_output,
hidden_states,
self.prefix,
)
return gated_output
def qwen4_exp_amd_ple_ngram_ids(
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
output: torch.Tensor,
layer_name: str,
) -> None:
"""Hash the current request layout into n-gram embedding indices.
The launch sizes its request binary search from the request count, which
is symbolic under Dynamo, so it runs outside Inductor's FX graph.
"""
layer = get_forward_context().no_compile_layers[layer_name]
if not isinstance(layer, Qwen4ExpPLELayer):
raise TypeError(f"{layer_name} is not a Qwen4Exp PLE owner")
embedding = layer.ple_embedding
ple_ngram_ids(
input_ids=input_ids,
query_start_loc=query_start_loc,
ngram_context=ngram_context,
layer_multipliers=embedding.layer_multipliers,
ngram_heads_vocab_sizes=embedding.ngram_heads_vocab_sizes,
ngram_heads_offsets=embedding.ngram_heads_offsets,
eos_token_id=embedding.eos_token_id,
heads_per_ngram=embedding.heads_per_ngram,
output=output,
)
def qwen4_exp_amd_ple_ngram_embedding(
ngram_ids: torch.Tensor,
output: torch.Tensor,
layer_name: str,
) -> None:
"""Run the large PLE embedding lookup outside Inductor's FX graph.
Keeping the embedding weight in ``static_forward_context`` prevents AOT
compile-time autotuning from materializing a synthetic copy of the weight.
"""
layer = get_forward_context().no_compile_layers[layer_name]
if not isinstance(layer, Qwen4ExpPLELayer):
raise TypeError(f"{layer_name} is not a Qwen4Exp PLE owner")
result = layer.ple_embedding.ngram_embedding(ngram_ids).flatten(-2)
output.copy_(result)
def qwen4_exp_amd_ple_ngram_embedding_pinned(
ngram_ids: torch.Tensor,
output: torch.Tensor,
layer_name: str,
) -> None:
"""Run the pinned PLE UVA lookup outside Inductor's FX graph.
Same rationale as the device-path escape: keeping the large embedding weight
out of the graph prevents AOT compile-time autotuning from materializing a
synthetic copy of the weight.
"""
layer = get_forward_context().no_compile_layers[layer_name]
if not isinstance(layer, Qwen4ExpPLELayer):
raise TypeError(f"{layer_name} is not a Qwen4Exp PLE owner")
result = layer.ple_embedding.ngram_embedding.sync_lookup(ngram_ids).flatten(-2)
output.copy_(result)
def qwen4_exp_ple_short_conv(
inputs: torch.Tensor,
residual: torch.Tensor,
outer_residual: torch.Tensor | None,
layer_name: str,
) -> None:
layer = get_forward_context().no_compile_layers[layer_name]
layer._short_conv(inputs, residual, outer_residual)
direct_register_custom_op(
op_name="qwen4_exp_amd_ple_ngram_ids",
op_func=qwen4_exp_amd_ple_ngram_ids,
mutates_args=["output"],
)
direct_register_custom_op(
op_name="qwen4_exp_amd_ple_ngram_embedding",
op_func=qwen4_exp_amd_ple_ngram_embedding,
mutates_args=["output"],
)
direct_register_custom_op(
op_name="qwen4_exp_amd_ple_ngram_embedding_pinned",
op_func=qwen4_exp_amd_ple_ngram_embedding_pinned,
mutates_args=["output"],
)
direct_register_custom_op(
op_name="qwen4_exp_ple_short_conv",
op_func=qwen4_exp_ple_short_conv,
mutates_args=["residual"],
)
__all__ = [
"Qwen4ExpNGramEmbedding",
"Qwen4ExpPLEGroupedNorm",
"Qwen4ExpPLELayer",
]
-544
View File
@@ -1,544 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""AMD ROCm QSA owner with Triton kernels."""
from __future__ import annotations
from typing import ClassVar, cast
import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.attention.attention import (
set_default_quant_scales,
)
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
from vllm.model_executor.models.qwen3_vl import mrope_positions_factor
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
canonicalize_singleton_dim_strides,
direct_register_custom_op,
kv_cache_dtype_str_to_dtype,
)
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionCGSupport,
AttentionType,
MultipleOf,
)
from vllm.v1.attention.backends.fa_utils import is_flash_attn_varlen_func_available
from vllm.v1.attention.backends.flash_attn import (
FlashAttentionBackend,
FlashAttentionImpl,
FlashAttentionMetadata,
FlashAttentionMetadataBuilder,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheSpec,
get_kv_quant_mode,
)
from ..common.qsa_cache import QSAForwardMetadata
from . import model
from .indexer_qsa import QSAIndexer
class Qwen4ExpQSAMetadataBuilder(FlashAttentionMetadataBuilder):
"""Flash metadata supporting uniform decode and target-verify graphs."""
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
"""FullAttentionSpec backend used by the merged QSA owner."""
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"]
@staticmethod
def get_name() -> str:
return "QWEN4_EXP_QSA_TRITON"
@staticmethod
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
# QSA consumes manager pages directly and does not use FA4 paged attention.
return [MultipleOf(16)]
@staticmethod
def get_impl_cls() -> type[Qwen4ExpQSAFlashAttentionImpl]:
return Qwen4ExpQSAFlashAttentionImpl
@staticmethod
def get_builder_cls() -> type[Qwen4ExpQSAMetadataBuilder]:
return Qwen4ExpQSAMetadataBuilder
@classmethod
def is_sparse(cls) -> bool:
return True
@classmethod
def supports_kv_connector(cls) -> bool:
return False
class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
"""Run paged sparse GQA with the QSA Triton kernel."""
supports_dcp: bool = False
supports_pcp: bool = False
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
if not is_flash_attn_varlen_func_available():
raise NotImplementedError("Qwen4Exp QSA requires FlashAttention")
if self.dcp_world_size != 1:
raise NotImplementedError(
"Qwen4Exp QSA does not support decode context parallelism"
)
if self.kv_cache_dtype not in ("auto", "bfloat16"):
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
self.supports_quant_query_input = False
def forward_qsa(
self,
layer: torch.nn.Module,
query: torch.Tensor,
key: torch.Tensor | None,
value: torch.Tensor | None,
kv_cache: torch.Tensor,
attn_metadata: FlashAttentionMetadata,
output: torch.Tensor,
token_to_req: torch.Tensor,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
) -> torch.Tensor:
del key, value
if output_scale is not None or output_block_scale is not None:
raise NotImplementedError("QSA does not support fused output quantization")
if self.alibi_slopes is not None or self.sinks is not None:
raise NotImplementedError("QSA does not support ALiBi or attention sinks")
if self.sliding_window != (-1, -1):
raise NotImplementedError("QSA does not support sliding-window attention")
num_tokens = attn_metadata.num_actual_tokens
output.zero_()
if num_tokens == 0:
return output
topk_buffer = getattr(layer, "topk_indices_buffer", None)
if topk_buffer is None:
raise RuntimeError("QSA owner did not provide its top-k buffer")
logical_indices = topk_buffer[:num_tokens]
token_to_req = token_to_req[:num_tokens]
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
key_cache = canonicalize_singleton_dim_strides(key_cache)
value_cache = canonicalize_singleton_dim_strides(value_cache)
if key_cache.dtype != torch.bfloat16 or query.dtype != torch.bfloat16:
raise NotImplementedError("Qwen4Exp QSA requires BF16 Q/K/V")
from .ops.qsa import qsa_sparse_paged_attention
qsa_sparse_paged_attention(
query[:num_tokens],
key_cache,
value_cache,
logical_indices,
attn_metadata.block_table,
token_to_req,
output[:num_tokens],
)
return output
class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
"""Merged Qwen full-attention owner with a QSA index side branch."""
supports_dcp = False
def __init__(
self,
*,
vllm_config: VllmConfig,
config: Qwen4ExpTextConfig,
layer_id: int,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
) -> None:
nn.Module.__init__(self)
cache_config = vllm_config.cache_config
model_config = vllm_config.model_config
if cache_config is None:
raise ValueError("Qwen4Exp QSA requires a paged KV cache")
if model_config.dtype != torch.bfloat16:
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
if cache_config.cache_dtype not in ("auto", "bfloat16"):
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
if getattr(quant_config, "kv_cache_scheme", None) is not None:
raise NotImplementedError("Qwen4Exp QSA does not support KV quantization")
parallel_config = vllm_config.parallel_config
if (
parallel_config.prefill_context_parallel_size > 1
or parallel_config.decode_context_parallel_size > 1
):
raise NotImplementedError(
"Qwen4Exp QSA does not support context parallelism"
)
if not getattr(config, "is_causal", True):
raise NotImplementedError("Qwen4Exp QSA requires causal decoder attention")
self.config = config
self.hidden_size = int(config.hidden_size)
tp_size = get_tensor_model_parallel_world_size()
self.total_num_heads = int(config.num_attention_heads)
if self.total_num_heads % tp_size:
raise ValueError("QSA attention heads must be divisible by TP size")
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = int(config.num_key_value_heads)
if self.total_num_kv_heads >= tp_size:
if self.total_num_kv_heads % tp_size:
raise ValueError("QSA KV heads must be divisible by TP size")
elif tp_size % self.total_num_kv_heads:
raise ValueError("TP size must be divisible by replicated QSA KV heads")
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.head_dim = int(config.head_dim or self.hidden_size // self.total_num_heads)
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = self.head_dim**-0.5
self.dual_chunk_attention_config = getattr(
config, "dual_chunk_attention_config", None
)
if self.dual_chunk_attention_config is not None:
raise NotImplementedError("Qwen4Exp QSA does not support dual-chunk RoPE")
# Qwen4Exp full-attention checkpoints always pack a sigmoid output
# gate next to Q, even when an inherited config default says otherwise.
self.attn_output_gate = True
self.qkv_proj = QKVParallelLinear(
self.hidden_size,
self.head_dim,
self.total_num_heads * (1 + self.attn_output_gate),
self.total_num_kv_heads,
bias=False,
quant_config=model.without_modelopt_fp4(quant_config),
prefix=f"{prefix}.qkv_proj",
)
self.o_proj = RowParallelLinear(
self.total_num_heads * self.head_dim,
self.hidden_size,
bias=False,
reduce_results=reduce_results,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
self.rotary_emb = get_rope(
head_size=self.head_dim,
max_position=config.max_position_embeddings,
rope_parameters=config.rope_parameters,
mrope_positions_factor=mrope_positions_factor(vllm_config),
)
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
mm_config = model_config.multimodal_config
text_only = mm_config is None or mm_config.language_model_only
self.use_fused_qk_norm_rope_gate = (
self.attn_output_gate
and getattr(self.rotary_emb, "is_neox_style", False)
and current_platform.is_cuda()
and text_only
)
self.layer_name = f"{prefix}.attn"
self.attn_type = AttentionType.DECODER
self.kv_cache_dtype = cache_config.cache_dtype
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
self.kv_cache_dtype, model_config
)
if self.kv_cache_torch_dtype != torch.bfloat16:
raise NotImplementedError("Qwen4Exp QSA requires BF16 cache storage")
self.kv_sharing_target_layer_name = None
self.kv_cache = torch.tensor([])
set_default_quant_scales(self, register_buffer=True)
self.attn_backend = Qwen4ExpQSAFlashAttentionBackend
self.impl = Qwen4ExpQSAFlashAttentionImpl(
self.num_heads,
self.head_dim,
self.scaling,
self.num_kv_heads,
None,
None,
self.kv_cache_dtype,
None,
AttentionType.DECODER,
None,
)
self.indexer = QSAIndexer(
vllm_config=vllm_config,
config=config,
layer_id=layer_id,
rotary_emb=self.rotary_emb,
quant_config=quant_config,
prefix=f"{prefix}.indexer",
)
self.use_fused_qsa_prepare = (
self.attn_output_gate
and self.indexer.use_fused_pre_indexer
and self.head_dim & (self.head_dim - 1) == 0
)
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
self.register_buffer(
"topk_indices_buffer",
torch.empty(
max_tokens,
self.indexer.output_width,
dtype=torch.int32,
),
persistent=False,
)
static_context = vllm_config.compilation_config.static_forward_context
if self.layer_name in static_context:
raise ValueError(f"Duplicate layer name: {self.layer_name}")
static_context[self.layer_name] = self
def get_attn_backend(self) -> type[AttentionBackend]:
return self.attn_backend
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
return FullAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_dim,
head_size_v=self.head_dim,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
)
def _run_qsa(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
query: torch.Tensor | None,
key: torch.Tensor | None,
value: torch.Tensor | None,
output: torch.Tensor,
qkv: torch.Tensor | None = None,
gate: torch.Tensor | None = None,
) -> None:
# query/key/value are None when the fused prepare runs inside the
# indexer launch, which then writes the gate into ``gate``.
metadata = get_forward_context().attn_metadata
if isinstance(metadata, list):
metadata = metadata[0]
if not isinstance(metadata, dict):
output.zero_()
if gate is not None:
gate.zero_()
return
main_metadata = cast(FlashAttentionMetadata, metadata[self.layer_name])
if self.kv_cache.numel() == 0:
raise RuntimeError("QSA main K/V cache is not bound")
num_tokens = main_metadata.num_actual_tokens
side_metadata = cast(
QSAForwardMetadata,
metadata[self.indexer.raw_key_cache.prefix],
)
if side_metadata.num_actual_tokens != num_tokens:
raise RuntimeError("QSA main and side metadata token counts disagree")
selected, main_outputs = self.indexer(
hidden_states,
positions,
self.topk_indices_buffer[:num_tokens],
attn=self,
qkv=qkv,
slot_mapping=main_metadata.slot_mapping,
gate_out=gate,
)
if selected.shape != (
num_tokens,
self.indexer.output_width,
):
raise RuntimeError("QSA indexer returned an invalid selection shape")
impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl)
if main_outputs is None:
assert key is not None and value is not None
impl.do_kv_cache_update(
self,
key,
value,
self.kv_cache,
main_metadata.slot_mapping,
)
else:
query, _ = main_outputs
# The gate is applied to every row outside this op; padded rows
# must stay finite so they keep the zero output forward_qsa gives.
assert gate is not None
gate[num_tokens:].zero_()
assert query is not None
impl.forward_qsa(
self,
query,
key,
value,
self.kv_cache,
main_metadata,
output,
token_to_req=side_metadata.token_to_req,
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
num_tokens = hidden_states.shape[0]
encoded_layer_name = _encode_layer_name(self.layer_name)
if self.use_fused_qsa_prepare:
# Norm/RoPE/gate and the K/V cache write happen inside the op.
attn_output = qkv.new_empty(num_tokens, self.num_heads, self.head_dim)
gate = torch.empty_like(attn_output)
if current_platform.opaque_attention_op():
torch.ops.vllm.qwen4_exp_qsa_prepare_with_output(
hidden_states,
positions,
qkv,
attn_output,
gate,
encoded_layer_name,
)
else:
qwen4_exp_qsa_prepare_with_output(
hidden_states,
positions,
qkv,
attn_output,
gate,
encoded_layer_name,
)
flat_output = attn_output.view(num_tokens, -1)
flat_output = flat_output * torch.sigmoid(gate.view(num_tokens, -1))
output, _ = self.o_proj(flat_output)
return output
q, k, v, gate = self._project_qkv_gate(qkv, positions)
query = q.view(num_tokens, self.num_heads, self.head_dim)
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
attn_output = torch.empty_like(query)
if current_platform.opaque_attention_op():
torch.ops.vllm.qwen4_exp_qsa_with_output(
hidden_states,
positions,
query,
key,
value,
attn_output,
encoded_layer_name,
)
else:
qwen4_exp_qsa_with_output(
hidden_states,
positions,
query,
key,
value,
attn_output,
encoded_layer_name,
)
flat_output = attn_output.view(num_tokens, -1)
if gate is not None:
flat_output = flat_output * torch.sigmoid(gate)
output, _ = self.o_proj(flat_output)
return output
def qwen4_exp_qsa_with_output(
hidden_states: torch.Tensor,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
output: torch.Tensor,
layer_name: LayerNameType,
) -> None:
"""Run the complete QSA state/update/attend transaction."""
layer_name = _resolve_layer_name(layer_name)
layer = get_forward_context().no_compile_layers[layer_name]
if not isinstance(layer, Qwen4ExpQSAAttention):
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
layer._run_qsa(
hidden_states,
positions,
query,
key,
value,
output,
)
direct_register_custom_op(
op_name="qwen4_exp_qsa_with_output",
op_func=qwen4_exp_qsa_with_output,
mutates_args=["output"],
)
def qwen4_exp_qsa_prepare_with_output(
hidden_states: torch.Tensor,
positions: torch.Tensor,
qkv: torch.Tensor,
output: torch.Tensor,
gate: torch.Tensor,
layer_name: LayerNameType,
) -> None:
"""Run the QSA transaction with the main QK-norm/RoPE/gate fused in."""
layer_name = _resolve_layer_name(layer_name)
layer = get_forward_context().no_compile_layers[layer_name]
if not isinstance(layer, Qwen4ExpQSAAttention):
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
layer._run_qsa(
hidden_states,
positions,
None,
None,
None,
output,
qkv=qkv,
gate=gate,
)
direct_register_custom_op(
op_name="qwen4_exp_qsa_prepare_with_output",
op_func=qwen4_exp_qsa_prepare_with_output,
mutates_args=["output", "gate"],
)
__all__ = [
"QSAIndexer",
"Qwen4ExpQSAAttention",
"Qwen4ExpQSAFlashAttentionBackend",
"Qwen4ExpQSAFlashAttentionImpl",
"qwen4_exp_qsa_prepare_with_output",
"qwen4_exp_qsa_with_output",
]