[Mini-VLLM] Repair test suite after SM86 cut
- Restore example connector, test helpers, and dropped patch tests - Guard MLA metadata import in the example connector - Point the PLE test at common.ops.ple after upstream #60021 move - Cut tests for removed features: tools, DRY, epd proxy, STT batch, diffusion scheduler, INC/Quark PLE cases, mamba-2 mixer Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,205 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HyperConnection (Gated Residual) utilities for the AMD model variant.
|
||||
|
||||
Implements the HyperConnection residual scheme proposed in
|
||||
"HyperConnections" (https://arxiv.org/abs/2409.19606). This AMD variant
|
||||
delays each HC combine to the following HC mix boundary. HC glue kernels,
|
||||
including fused combine+RMSNorm, live in ``ops/hc.py``; projections remain
|
||||
standard vLLM Linear modules.
|
||||
|
||||
Hidden states between layers have shape ``[..., HC*HS]`` with HS inner
|
||||
(HC outer, HS inner — checkpoint-native layout).
|
||||
|
||||
Typical usage inside a transformer decoder layer::
|
||||
|
||||
self.attn_hc = GatedResidual(hc_config)
|
||||
|
||||
hidden_states, block_input, injection = self.attn_hc.mix(hidden_states)
|
||||
attention_output = attention(block_input)
|
||||
hidden_states, block_input, injection = self.mlp_hc.combine_and_mix(
|
||||
hidden_states, attention_output, injection
|
||||
)
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
|
||||
from ..common.hyperconnection import (
|
||||
GroupedGemmaRMSNorm,
|
||||
HyperConnectionConfig,
|
||||
)
|
||||
from .ops.hc import (
|
||||
grouped_gemma_rmsnorm,
|
||||
hc_combine,
|
||||
hc_combine_norm,
|
||||
hc_gate_mix,
|
||||
hc_silu,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gated-residual variant
|
||||
# ---------------------------------------------------------------------------
|
||||
class GatedResidual(nn.Module):
|
||||
"""Gated HyperConnection with learnable low-rank mixing and injection.
|
||||
|
||||
``combine_and_mix()`` runs the pre pipeline (grouped GemmaRMSNorm -> merged
|
||||
low-rank down+inject GEMM -> silu -> up GEMM -> sigmoid -> gated mean
|
||||
over the HC streams). When passed a pending block output and an injection,
|
||||
it fuses their residual combine with the RMSNorm. Final mixers use
|
||||
``use_combine=False`` and do not produce a new injection.
|
||||
|
||||
Weights: the norm owns the grouped GemmaRMSNorm affine; the projections
|
||||
are vLLM Linear modules (merged replicated linear for down+inject), so
|
||||
GEMM dispatch (e.g. the low-latency skinny GEMM) applies through the
|
||||
standard quant_method mechanism.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: HyperConnectionConfig,
|
||||
use_combine: bool = True,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.lora_rank = config.hc_lowrank
|
||||
self.hc_count = config.hc_count
|
||||
self.hidden_size = config.hidden_size
|
||||
self.use_combine = use_combine
|
||||
|
||||
norm_size = (
|
||||
self.hyper_hidden_size if config.hc_per_branch_norm else config.hidden_size
|
||||
)
|
||||
group_size = config.hidden_size if config.hc_per_branch_norm else None
|
||||
# Normalize each H-sized HC stream independently while retaining a
|
||||
# separate affine weight for every element of the HC*H layout.
|
||||
self.hc_norm = GroupedGemmaRMSNorm(
|
||||
norm_size,
|
||||
eps=config.rms_norm_eps,
|
||||
group_size=group_size,
|
||||
dtype=config.params_dtype,
|
||||
)
|
||||
|
||||
# -- vLLM Linear weights --------------------------------------------
|
||||
# The merged skinny-GEMM shape is physically padded to 16 rows for
|
||||
# alignment and efficient backend dispatch.
|
||||
self.pad_size = (-(self.lora_rank + self.hc_count)) % 16 if use_combine else 0
|
||||
if use_combine:
|
||||
self.input_mix_weight_down_block_inject = MergedColumnParallelLinear(
|
||||
self.hyper_hidden_size,
|
||||
[self.lora_rank, self.hc_count]
|
||||
+ ([self.pad_size] if self.pad_size else []),
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "input_mix_weight_down_block_inject"),
|
||||
return_bias=False,
|
||||
disable_tp=True,
|
||||
)
|
||||
else:
|
||||
self.input_mix_weight_down = ReplicatedLinear(
|
||||
self.hyper_hidden_size,
|
||||
self.lora_rank,
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "input_mix_weight_down"),
|
||||
return_bias=False,
|
||||
)
|
||||
self.input_mix_weight_up = ReplicatedLinear(
|
||||
self.lora_rank,
|
||||
self.hyper_hidden_size,
|
||||
bias=False,
|
||||
params_dtype=config.params_dtype,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "input_mix_weight_up"),
|
||||
return_bias=False,
|
||||
)
|
||||
|
||||
def mix(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
xn = grouped_gemma_rmsnorm(
|
||||
hidden_states,
|
||||
self.hc_norm.weight,
|
||||
self.config.rms_norm_eps,
|
||||
self.hc_count,
|
||||
)
|
||||
|
||||
if self.use_combine:
|
||||
# produce injection logits for combine
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
else:
|
||||
lora = self.input_mix_weight_down(xn)
|
||||
injection = None
|
||||
|
||||
lora = hc_silu(lora, self.hc_count)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
return hidden_states, block_input, injection
|
||||
|
||||
def combine_and_mix(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prev_block_output: torch.Tensor,
|
||||
prev_injection: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""Consume a pending combine, then prepare the next block input.
|
||||
|
||||
``hidden_states`` is the multi-stream state from before the pending
|
||||
block's mix. Its combine with ``block_output`` is fused with this
|
||||
module's input RMSNorm.
|
||||
"""
|
||||
hidden_states, xn = hc_combine_norm(
|
||||
hidden_states,
|
||||
prev_block_output,
|
||||
prev_injection,
|
||||
self.hc_norm.weight,
|
||||
self.config.rms_norm_eps,
|
||||
self.hc_count,
|
||||
)
|
||||
|
||||
if self.use_combine:
|
||||
# produce injection logits for combine
|
||||
split_sizes = [self.lora_rank, self.hc_count, self.pad_size]
|
||||
down_and_injection = self.input_mix_weight_down_block_inject(xn)
|
||||
lora, injection, _ = down_and_injection.split(split_sizes, dim=-1)
|
||||
else:
|
||||
lora = self.input_mix_weight_down(xn)
|
||||
injection = None
|
||||
|
||||
lora = hc_silu(lora, self.hc_count)
|
||||
gate = self.input_mix_weight_up(lora) # [M, D]
|
||||
block_input = hc_gate_mix(xn, gate, self.hc_count)
|
||||
|
||||
return hidden_states, block_input, injection
|
||||
|
||||
def combine(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return hc_combine(hidden_states, block_output, injection, self.hc_count)
|
||||
|
||||
@property
|
||||
def hyper_hidden_size(self) -> int:
|
||||
return self.hc_count * self.hidden_size
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GatedResidual",
|
||||
"GroupedGemmaRMSNorm",
|
||||
"HyperConnectionConfig",
|
||||
]
|
||||
@@ -0,0 +1,420 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen4Exp weight-free QSA indexer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
|
||||
|
||||
from ..common.qsa_cache import (
|
||||
QSACompressedKeyCache,
|
||||
QSAForwardMetadata,
|
||||
QSAKeyStateCache,
|
||||
canonical_qsa_rope_positions,
|
||||
)
|
||||
from .ops.qsa_prepare import qsa_prepare, supports_fused_pre_indexer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .qsa import Qwen4ExpQSAAttention
|
||||
|
||||
|
||||
def apply_qsa_rope(
|
||||
rotary_emb: nn.Module,
|
||||
positions: torch.Tensor,
|
||||
tensor: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the main attention's exact 1D/MRoPE composition to QSA heads."""
|
||||
num_tokens, _, head_dim = tensor.shape
|
||||
rotary_dim = rotary_emb.rotary_dim
|
||||
cache = rotary_emb._match_cos_sin_cache_dtype(tensor) # noqa: SLF001
|
||||
cos_sin = cache[positions]
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
if positions.ndim == 2:
|
||||
shape = tensor.shape
|
||||
tensor, _ = triton_mrope(
|
||||
tensor.reshape(num_tokens, -1),
|
||||
tensor.new_empty((num_tokens, head_dim)),
|
||||
cos,
|
||||
sin,
|
||||
rotary_emb.mrope_section,
|
||||
head_dim,
|
||||
rotary_dim,
|
||||
rotary_emb.mrope_interleaved,
|
||||
rotary_emb.is_neox_style,
|
||||
)
|
||||
return tensor.reshape(shape)
|
||||
|
||||
rotated = rotary_emb.apply_rotary_emb(
|
||||
tensor[..., :rotary_dim],
|
||||
cos,
|
||||
sin,
|
||||
)
|
||||
return torch.cat((rotated, tensor[..., rotary_dim:]), dim=-1)
|
||||
|
||||
|
||||
def apply_qsa_rmsnorm(
|
||||
norm: GemmaRMSNorm,
|
||||
tensor: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Use vLLM's portable RMSNorm implementation on ROCm."""
|
||||
return cast(torch.Tensor, norm(tensor))
|
||||
|
||||
|
||||
class QSAIndexer(nn.Module):
|
||||
"""Replicated Q/K projection plus paged, weight-free QSA selection.
|
||||
|
||||
``prefix`` must be the checkpoint's indexer prefix, normally
|
||||
``model.layers.N.self_attn.indexer``. Consequently the trainable names are
|
||||
``index_qk_proj``, ``q_layernorm`` and ``k_layernorm`` under that prefix.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
config: Qwen4ExpTextConfig,
|
||||
layer_id: int,
|
||||
rotary_emb: nn.Module,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if vllm_config.cache_config is None:
|
||||
raise ValueError("QSA requires a paged KV cache")
|
||||
if vllm_config.model_config.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
|
||||
|
||||
self.layer_id = int(layer_id)
|
||||
self.index_n_heads = int(config.indexer_n_heads)
|
||||
self.index_kv_heads = int(config.indexer_kv_heads)
|
||||
self.index_head_dim = int(config.indexer_head_dim)
|
||||
self.token_topk = int(config.indexer_budget)
|
||||
self.compress_ratio = int(config.indexer_compress_ratio)
|
||||
self.rotary_emb = rotary_emb
|
||||
self.use_fused_pre_indexer = supports_fused_pre_indexer(
|
||||
rotary_emb,
|
||||
self.index_head_dim,
|
||||
self.index_kv_heads,
|
||||
self.compress_ratio,
|
||||
)
|
||||
self.prefix = prefix
|
||||
# MTP step 0 selects the target-aligned rows; later steps reuse them
|
||||
# while continuing to update the QSA side cache.
|
||||
self.skip_topk = False
|
||||
|
||||
self.index_qk_proj = ReplicatedLinear(
|
||||
int(config.hidden_size),
|
||||
(self.index_n_heads + self.index_kv_heads) * self.index_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.index_qk_proj" if prefix else "index_qk_proj",
|
||||
)
|
||||
self.q_layernorm = GemmaRMSNorm(
|
||||
self.index_head_dim,
|
||||
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
|
||||
)
|
||||
self.k_layernorm = GemmaRMSNorm(
|
||||
self.index_head_dim,
|
||||
eps=float(getattr(config, "rms_norm_eps", 1e-6)),
|
||||
)
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
cache_prefix = f"{prefix}." if prefix else ""
|
||||
self.raw_key_cache = QSAKeyStateCache(
|
||||
head_size=self.index_head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
cache_rope_positions=vllm_config.model_config.uses_mrope,
|
||||
prefix=f"{cache_prefix}raw_key_cache",
|
||||
cache_config=cache_config,
|
||||
compress_ratio=self.compress_ratio,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
self.compressed_key_cache = QSACompressedKeyCache(
|
||||
head_size=self.index_head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
compress_ratio=self.compress_ratio,
|
||||
prefix=f"{cache_prefix}compressed_key_cache",
|
||||
cache_config=cache_config,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
@property
|
||||
def output_width(self) -> int:
|
||||
return self.token_topk + self.compress_ratio - 1
|
||||
|
||||
def project(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Project replicated Q/K and split, leaving both operands unrotated."""
|
||||
qk, _ = self.index_qk_proj(hidden_states)
|
||||
return qk.split(
|
||||
(
|
||||
self.index_n_heads * self.index_head_dim,
|
||||
self.index_kv_heads * self.index_head_dim,
|
||||
),
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
def project_qk(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Project replicated Q/K, normalize+rotate Q, and preserve raw K."""
|
||||
q_raw, token_k = self.project(hidden_states)
|
||||
q = q_raw.reshape(-1, self.index_n_heads, self.index_head_dim)
|
||||
q = apply_qsa_rmsnorm(
|
||||
self.q_layernorm,
|
||||
q.reshape(-1, self.index_head_dim),
|
||||
).reshape_as(q)
|
||||
q = apply_qsa_rope(self.rotary_emb, positions, q)
|
||||
return q, token_k.reshape(-1, 1, self.index_head_dim)
|
||||
|
||||
def normalize_compressed_keys(
|
||||
self,
|
||||
compressed_keys: torch.Tensor,
|
||||
first_rope_positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Normalize pooled K and apply the first token's exact group position."""
|
||||
keys = compressed_keys.reshape(-1, self.index_head_dim)
|
||||
keys = apply_qsa_rmsnorm(self.k_layernorm, keys).reshape(
|
||||
-1, 1, self.index_head_dim
|
||||
)
|
||||
if getattr(self.rotary_emb, "mrope_section", None):
|
||||
positions = first_rope_positions.transpose(0, 1)
|
||||
else:
|
||||
positions = first_rope_positions[:, 0]
|
||||
return apply_qsa_rope(self.rotary_emb, positions, keys)
|
||||
|
||||
def _metadata(
|
||||
self,
|
||||
) -> tuple[QSAForwardMetadata, QSAForwardMetadata] | None:
|
||||
metadata = get_forward_context().attn_metadata
|
||||
if isinstance(metadata, list):
|
||||
metadata = metadata[0]
|
||||
if not isinstance(metadata, dict):
|
||||
return None
|
||||
raw = cast(QSAForwardMetadata, metadata[self.raw_key_cache.prefix])
|
||||
compressed = cast(
|
||||
QSAForwardMetadata, metadata[self.compressed_key_cache.prefix]
|
||||
)
|
||||
if raw.num_actual_tokens != compressed.num_actual_tokens:
|
||||
raise RuntimeError("QSA side-cache metadata token counts disagree")
|
||||
if not raw.logical_positions.is_cuda and (
|
||||
not torch.equal(raw.logical_positions, compressed.logical_positions)
|
||||
):
|
||||
raise RuntimeError("QSA side-cache metadata positions disagree")
|
||||
return raw, compressed
|
||||
|
||||
def _update_and_compress(
|
||||
self,
|
||||
token_k: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
raw_metadata: QSAForwardMetadata,
|
||||
compressed_metadata: QSAForwardMetadata,
|
||||
) -> None:
|
||||
num_tokens = raw_metadata.num_actual_tokens
|
||||
raw_key_cache = self.raw_key_cache.key_cache
|
||||
rope_position_cache = self.raw_key_cache.rope_position_cache
|
||||
from .ops.qsa import qsa_compress_groups_with_ratio, qsa_store_cache_rows
|
||||
|
||||
if rope_position_cache is None:
|
||||
position_rows = raw_metadata.logical_positions.view(-1, 1, 1).expand(
|
||||
-1, 1, 3
|
||||
)
|
||||
else:
|
||||
position_rows = canonical_qsa_rope_positions(positions)[:num_tokens].to(
|
||||
device=raw_key_cache.device
|
||||
)
|
||||
pooled, first_positions = qsa_compress_groups_with_ratio(
|
||||
token_k[:num_tokens],
|
||||
position_rows,
|
||||
raw_key_cache,
|
||||
raw_metadata.block_table,
|
||||
raw_metadata.token_to_req,
|
||||
raw_metadata.query_start_loc,
|
||||
raw_metadata.logical_positions,
|
||||
compressed_metadata.slot_mapping,
|
||||
self.compress_ratio,
|
||||
rope_position_cache,
|
||||
)
|
||||
normalized = self.normalize_compressed_keys(pooled, first_positions)
|
||||
qsa_store_cache_rows(
|
||||
self.compressed_key_cache.kv_cache,
|
||||
compressed_metadata.slot_mapping,
|
||||
normalized,
|
||||
)
|
||||
qsa_store_cache_rows(
|
||||
raw_key_cache,
|
||||
raw_metadata.slot_mapping,
|
||||
token_k[:num_tokens],
|
||||
)
|
||||
if rope_position_cache is not None:
|
||||
qsa_store_cache_rows(
|
||||
rope_position_cache,
|
||||
raw_metadata.slot_mapping,
|
||||
position_rows,
|
||||
)
|
||||
|
||||
def _fused_prepare(
|
||||
self,
|
||||
projected_q: torch.Tensor,
|
||||
raw_keys: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
raw_metadata: QSAForwardMetadata,
|
||||
compressed_metadata: QSAForwardMetadata,
|
||||
attn: Qwen4ExpQSAAttention,
|
||||
qkv: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
gate_out: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""Prepare the indexer and ``attn``'s main attention in one launch.
|
||||
|
||||
Normalizes and rotates the index Q, compresses K and writes both side
|
||||
caches, which replaces ``project_qk``'s norm/RoPE half and the whole of
|
||||
``_update_and_compress``. The same launch applies ``attn``'s QK-norm and
|
||||
RoPE, copies its gate and writes its K/V into ``attn.kv_cache``,
|
||||
replacing ``_project_qkv_gate`` and ``do_kv_cache_update``. Both halves
|
||||
must stay numerically interchangeable with what they replace.
|
||||
"""
|
||||
num_tokens = raw_metadata.num_actual_tokens
|
||||
raw_key_cache = self.raw_key_cache
|
||||
q = projected_q.new_empty(num_tokens, self.index_n_heads, self.index_head_dim)
|
||||
main_outputs = qsa_prepare(
|
||||
projected_q,
|
||||
raw_keys,
|
||||
positions,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.q_layernorm.weight,
|
||||
self.k_layernorm.weight,
|
||||
self.q_layernorm.variance_epsilon,
|
||||
q,
|
||||
raw_key_cache.kv_cache,
|
||||
raw_metadata.slot_mapping,
|
||||
raw_metadata.block_table,
|
||||
raw_metadata.query_start_loc,
|
||||
raw_metadata.logical_positions,
|
||||
self.compressed_key_cache.kv_cache,
|
||||
compressed_metadata.slot_mapping,
|
||||
compressed_metadata.k_work_metadata,
|
||||
compress_ratio=self.compress_ratio,
|
||||
mrope_section=getattr(self.rotary_emb, "mrope_section", None),
|
||||
rope_pos_offset=(
|
||||
raw_key_cache.rope_position_offset
|
||||
if raw_key_cache.cache_rope_positions
|
||||
else None
|
||||
),
|
||||
main_qkv=qkv[:num_tokens],
|
||||
main_q_norm_weight=attn.q_norm.weight,
|
||||
main_k_norm_weight=attn.k_norm.weight,
|
||||
main_eps=attn.q_norm.variance_epsilon,
|
||||
main_kv_cache=attn.kv_cache.transpose(1, 2),
|
||||
main_slot_mapping=slot_mapping[:num_tokens],
|
||||
main_k_scale=attn._k_scale_float,
|
||||
main_v_scale=attn._v_scale_float,
|
||||
main_gate_out=None if gate_out is None else gate_out[:num_tokens],
|
||||
)
|
||||
return q, main_outputs
|
||||
|
||||
def _select(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
metadata: QSAForwardMetadata,
|
||||
out: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
from .ops.qsa import qsa_select_paged_tokens
|
||||
|
||||
return qsa_select_paged_tokens(
|
||||
q,
|
||||
self.compressed_key_cache.kv_cache,
|
||||
metadata.block_table,
|
||||
metadata.token_to_req,
|
||||
metadata.logical_positions,
|
||||
metadata.seq_lens,
|
||||
self.token_topk,
|
||||
self.compress_ratio,
|
||||
out,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
out: torch.Tensor | None = None,
|
||||
*,
|
||||
attn: Qwen4ExpQSAAttention,
|
||||
qkv: torch.Tensor | None = None,
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
gate_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]:
|
||||
"""Return fixed-width request-relative token indices padded with ``-1``.
|
||||
|
||||
With ``attn.use_fused_qsa_prepare``, the same launch also writes
|
||||
``attn``'s K/V into ``attn.kv_cache`` at ``slot_mapping`` and prepares
|
||||
its Q and gate from ``qkv``, returned as the second element (None
|
||||
otherwise). The gate is written into ``gate_out`` when given. ``qkv``,
|
||||
``slot_mapping`` and ``gate_out`` are only read in that mode.
|
||||
"""
|
||||
metadata = self._metadata()
|
||||
if metadata is None:
|
||||
# Preserve step-0 indices when later MTP steps reuse the buffer.
|
||||
if self.skip_topk and out is not None:
|
||||
return out, None
|
||||
result = torch.full(
|
||||
(hidden_states.shape[0], self.output_width),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
if out is not None:
|
||||
out.copy_(result)
|
||||
return out, None
|
||||
return result, None
|
||||
raw_metadata, compressed_metadata = metadata
|
||||
num_tokens = raw_metadata.num_actual_tokens
|
||||
hidden_states = hidden_states[:num_tokens]
|
||||
positions = positions[..., :num_tokens]
|
||||
main_outputs: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
if attn.use_fused_qsa_prepare:
|
||||
if qkv is None or slot_mapping is None:
|
||||
raise ValueError("fused QSA prepare requires qkv and slot_mapping")
|
||||
projected_q, raw_keys = self.project(hidden_states)
|
||||
q, main_outputs = self._fused_prepare(
|
||||
projected_q,
|
||||
raw_keys,
|
||||
positions,
|
||||
raw_metadata,
|
||||
compressed_metadata,
|
||||
attn,
|
||||
qkv,
|
||||
slot_mapping,
|
||||
gate_out,
|
||||
)
|
||||
else:
|
||||
q, token_k = self.project_qk(hidden_states, positions)
|
||||
self._update_and_compress(
|
||||
token_k,
|
||||
positions,
|
||||
raw_metadata,
|
||||
compressed_metadata,
|
||||
)
|
||||
if self.skip_topk:
|
||||
if out is None:
|
||||
raise RuntimeError("QSA top-k reuse requires an output buffer")
|
||||
return out, main_outputs
|
||||
return self._select(q, compressed_metadata, out), main_outputs
|
||||
|
||||
|
||||
__all__ = ["QSAIndexer", "apply_qsa_rope"]
|
||||
@@ -0,0 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen4Exp low-latency GEMM hook for AMD ROCm."""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
|
||||
def enable_qwen4_exp_low_latency_gemm(
|
||||
module: nn.Module,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Keep the standard vLLM linear methods on AMD ROCm."""
|
||||
del module, dtype
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,143 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Model-runner state for Qwen4Exp PLE inputs."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
|
||||
class Qwen4ExpModelState(MambaHybridModelState):
|
||||
"""Add rollback-safe PLE n-gram context to the model inputs."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
model: nn.Module,
|
||||
encoder_cache: EncoderCache | None,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(vllm_config, model, encoder_cache, device)
|
||||
config = self.model_config.hf_text_config
|
||||
self.uses_ngram_embedding = bool(config.ple_layer_ids)
|
||||
if not self.uses_ngram_embedding:
|
||||
self.ngram_context_len = 0
|
||||
self.ngram_eos_token_id = 0
|
||||
return
|
||||
|
||||
if vllm_config.parallel_config.pipeline_parallel_size > 1:
|
||||
raise RuntimeError(
|
||||
"N-gram PLE embedding currently requires "
|
||||
"pipeline_parallel_size=1 because non-first pipeline ranks do "
|
||||
"not receive the raw input_ids required by PLE. Please run "
|
||||
"with PP=1."
|
||||
)
|
||||
|
||||
self.ngram_context_len = int(config.ngram_size) - 1
|
||||
if self.ngram_context_len <= 0:
|
||||
raise ValueError("N-gram embedding requires context length >= 1.")
|
||||
self.ngram_eos_token_id = int(config.eos_token_id)
|
||||
self.ngram_context = torch.full(
|
||||
(self.max_num_reqs, self.ngram_context_len),
|
||||
self.ngram_eos_token_id,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self.ngram_context_offsets = torch.arange(
|
||||
-self.ngram_context_len,
|
||||
0,
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
self.ple_query_start_loc = torch.zeros(
|
||||
self.max_num_reqs + 1,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def _prepare_ngram_context(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> torch.Tensor:
|
||||
num_reqs = input_batch.num_reqs
|
||||
num_reqs_padded = input_batch.num_reqs_after_padding
|
||||
context = self.ngram_context[:num_reqs_padded]
|
||||
context.fill_(self.ngram_eos_token_id)
|
||||
if num_reqs == 0:
|
||||
return context
|
||||
|
||||
request_indices = input_batch.idx_mapping[:num_reqs].long()
|
||||
context_end = req_states.num_computed_tokens.gpu[request_indices].long()
|
||||
token_indices = context_end.unsqueeze(1) + self.ngram_context_offsets
|
||||
valid_tokens = token_indices >= 0
|
||||
token_indices.clamp_min_(0)
|
||||
context_tokens = req_states.all_token_ids.gpu[
|
||||
request_indices.unsqueeze(1), token_indices
|
||||
]
|
||||
context[:num_reqs].copy_(
|
||||
torch.where(
|
||||
valid_tokens,
|
||||
context_tokens,
|
||||
context_tokens.new_full((), self.ngram_eos_token_id),
|
||||
)
|
||||
)
|
||||
return context
|
||||
|
||||
def prepare_inputs(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> dict[str, Any]:
|
||||
model_inputs = super().prepare_inputs(input_batch, req_states)
|
||||
if not self.uses_ngram_embedding:
|
||||
return model_inputs
|
||||
|
||||
num_reqs_padded = input_batch.num_reqs_after_padding
|
||||
query_start_loc = self.ple_query_start_loc[: num_reqs_padded + 1]
|
||||
query_start_loc.copy_(input_batch.query_start_loc[: num_reqs_padded + 1])
|
||||
model_inputs.update(
|
||||
query_start_loc=query_start_loc,
|
||||
ngram_context=self._prepare_ngram_context(input_batch, req_states),
|
||||
)
|
||||
return model_inputs
|
||||
|
||||
def prepare_dummy_inputs(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
) -> dict[str, Any]:
|
||||
model_inputs = super().prepare_dummy_inputs(num_reqs, num_tokens)
|
||||
if not self.uses_ngram_embedding:
|
||||
return model_inputs
|
||||
|
||||
query_start_loc = self.ple_query_start_loc[: num_reqs + 1]
|
||||
query_start_loc[0] = 0
|
||||
tokens_per_req, num_extra_tokens = divmod(num_tokens, num_reqs)
|
||||
query_lens = torch.full(
|
||||
(num_reqs,),
|
||||
tokens_per_req,
|
||||
dtype=query_start_loc.dtype,
|
||||
device=query_start_loc.device,
|
||||
)
|
||||
if num_extra_tokens > 0:
|
||||
query_lens[-num_extra_tokens:] += 1
|
||||
torch.cumsum(query_lens, dim=0, out=query_start_loc[1:])
|
||||
|
||||
ngram_context = self.ngram_context[:num_reqs]
|
||||
ngram_context.fill_(self.ngram_eos_token_id)
|
||||
model_inputs.update(
|
||||
query_start_loc=query_start_loc,
|
||||
ngram_context=ngram_context,
|
||||
)
|
||||
return model_inputs
|
||||
|
||||
|
||||
__all__ = ["Qwen4ExpModelState"]
|
||||
@@ -0,0 +1,495 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Qwen4Exp MTP (Multi-Token Predictor) model.
|
||||
|
||||
The MTP draft model reuses the Qwen4Exp backbone (PLE/HC/MoE) but:
|
||||
- drops all multi-modal handling (text-only),
|
||||
- forces PLE off while keeping the main model's HC stream count,
|
||||
- fuses the backbone hidden and the new-token embedding via
|
||||
``residual_linear_shared`` (fc_embedding + shared fc_hidden) instead of
|
||||
the ``Linear(2H, H)`` + repeat used by other MTP variants,
|
||||
- emits TWO hidden streams per step (scheme A): a single stream [T, H]
|
||||
(final-mixer collapsed, fed to the LM head) and a pre-final-mixer
|
||||
multi stream [T, hc_count*H] (fed to the next draft step).
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig, replace, set_current_vllm_config
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.utils import configure_quant_config
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
LocalArgmaxMixin,
|
||||
SupportsHCSP,
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
get_draft_quant_config,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.common.ops.sequence_parallel import sp_all_gather, sp_shard
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from ..common.hyperconnection import is_hc_sequence_parallel_enabled
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .model import (
|
||||
_EXTRA_WEIGHTS_MAPPER,
|
||||
_QWEN4_EXP_IGNORED_MISSING_SUFFIXES,
|
||||
Qwen4ExpDecoderLayer,
|
||||
Qwen4ExpMixtureOfExperts,
|
||||
Qwen4ExpSparseMoeBlock,
|
||||
)
|
||||
|
||||
|
||||
def _remap_ignored_layers(
|
||||
ignored_layers: list[str],
|
||||
mtp_start_layer_idx: int,
|
||||
) -> list[str]:
|
||||
remapped: list[str] = []
|
||||
for name in ignored_layers:
|
||||
if name.startswith("mtp."):
|
||||
new_name = re.sub(
|
||||
r"(?<=\.layers\.)\d+",
|
||||
lambda m: str(mtp_start_layer_idx + int(m.group(0))),
|
||||
name,
|
||||
)
|
||||
remapped.append(new_name)
|
||||
else:
|
||||
remapped.append(name)
|
||||
return remapped
|
||||
|
||||
|
||||
def _remap_quantized_layers(
|
||||
quantized_layers: dict[str, dict],
|
||||
mtp_start_layer_idx: int,
|
||||
) -> dict[str, dict]:
|
||||
"""Map checkpoint MTP layer indices to standalone draft indices."""
|
||||
return {
|
||||
_remap_ignored_layers([name], mtp_start_layer_idx)[0]: layer_info
|
||||
for name, layer_info in quantized_layers.items()
|
||||
}
|
||||
|
||||
|
||||
def _remap_mtp_weight_name(name: str) -> str | None:
|
||||
"""Map Qwen4Exp checkpoint paths into the standalone draft model."""
|
||||
for checkpoint_prefix in (
|
||||
"model.language_model.",
|
||||
"language_model.",
|
||||
):
|
||||
if name.startswith(checkpoint_prefix):
|
||||
name = name.removeprefix(checkpoint_prefix)
|
||||
break
|
||||
|
||||
if name.startswith("embed_tokens."):
|
||||
name = f"model.{name}"
|
||||
if name.startswith("model.mtp."):
|
||||
name = name.removeprefix("model.")
|
||||
if name.startswith("mtp.shared_head.head."):
|
||||
return name.replace("mtp.shared_head.head.", "lm_head.", 1)
|
||||
if name.startswith("model.shared_head.head."):
|
||||
return name.replace("model.shared_head.head.", "lm_head.", 1)
|
||||
if name.startswith("shared_head.head."):
|
||||
return name.replace("shared_head.head.", "lm_head.", 1)
|
||||
if name.startswith("model.lm_head."):
|
||||
return name.removeprefix("model.")
|
||||
if name.startswith("mtp."):
|
||||
return name.replace("mtp.", "model.", 1)
|
||||
if name.startswith("model.embed_tokens.") or name.startswith("lm_head."):
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
def _make_draft_vllm_config(
|
||||
vllm_config: VllmConfig,
|
||||
mtp_start_layer_idx: int,
|
||||
) -> VllmConfig:
|
||||
"""Ensure that the draft model config is set in the vLLM config."""
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None or speculative_config.draft_model_config is None:
|
||||
raise ValueError("speculative_config.draft_model_config must be set")
|
||||
|
||||
draft_quant_config = get_draft_quant_config(vllm_config)
|
||||
|
||||
# inject packed and ignored modules to the quantization config of draft model
|
||||
if draft_quant_config is not None:
|
||||
configure_quant_config(draft_quant_config, Qwen4ExpMTP)
|
||||
ignored_layers = getattr(draft_quant_config, "ignored_layers", None)
|
||||
if ignored_layers:
|
||||
setattr( # noqa: B010
|
||||
draft_quant_config,
|
||||
"ignored_layers",
|
||||
_remap_ignored_layers(ignored_layers, mtp_start_layer_idx),
|
||||
)
|
||||
exclude_modules = getattr(draft_quant_config, "exclude_modules", None)
|
||||
if exclude_modules:
|
||||
setattr( # noqa: B010
|
||||
draft_quant_config,
|
||||
"exclude_modules",
|
||||
_remap_ignored_layers(exclude_modules, mtp_start_layer_idx),
|
||||
)
|
||||
quantized_layers = getattr(draft_quant_config, "quantized_layers", None)
|
||||
if quantized_layers:
|
||||
setattr( # noqa: B010
|
||||
draft_quant_config,
|
||||
"quantized_layers",
|
||||
_remap_quantized_layers(quantized_layers, mtp_start_layer_idx),
|
||||
)
|
||||
|
||||
draft_vllm_config = replace(
|
||||
vllm_config,
|
||||
model_config=speculative_config.draft_model_config,
|
||||
)
|
||||
# VllmConfig post-init derives the target quant config, so restore the
|
||||
# independently resolved draft quant config after replacement.
|
||||
draft_vllm_config.quant_config = draft_quant_config
|
||||
return draft_vllm_config
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
"hidden_states": 0,
|
||||
}
|
||||
)
|
||||
class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
config: Qwen4ExpTextConfig = model_config.hf_text_config
|
||||
|
||||
self.config = config
|
||||
# MoE SP also enables HC SP, while HC SP can be enabled independently.
|
||||
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
self.num_mtp_layers = getattr(config, "mtp_num_hidden_layers", 1)
|
||||
|
||||
self.hidden_size = config.hidden_size
|
||||
self.hc_count = config.hc_count
|
||||
|
||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, self.hidden_size)
|
||||
draft_vllm_config = _make_draft_vllm_config(
|
||||
vllm_config,
|
||||
self.mtp_start_layer_idx,
|
||||
)
|
||||
with set_current_vllm_config(draft_vllm_config, prefix=prefix):
|
||||
# residual_linear_shared fusion: fc_embedding projects the token
|
||||
# embedding, fc_hidden (shared across HC branches) projects the
|
||||
# backbone hidden; the embedding is added as a residual to every
|
||||
# branch (see mtp_residual_linear_shared.md).
|
||||
self.fc_embedding = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_size,
|
||||
gather_output=True,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=draft_vllm_config.quant_config,
|
||||
prefix=f"{prefix}.fc_embedding",
|
||||
)
|
||||
self.fc_hidden = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_size,
|
||||
gather_output=True,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=draft_vllm_config.quant_config,
|
||||
prefix=f"{prefix}.fc_hidden",
|
||||
)
|
||||
self.layers = nn.ModuleList(
|
||||
Qwen4ExpDecoderLayer(
|
||||
draft_vllm_config,
|
||||
layer_type="qwen_sparse_attention",
|
||||
prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
Qwen4ExpSparseMoeBlock,
|
||||
"mlp",
|
||||
)
|
||||
|
||||
self.pre_fc_norm_embedding = GemmaRMSNorm(
|
||||
self.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.pre_fc_norm_hidden = GemmaRMSNorm(
|
||||
self.hidden_size * self.hc_count, eps=config.rms_norm_eps
|
||||
)
|
||||
# HC final mixer collapses the multi stream into [T, H] for the LM head.
|
||||
hc_config = HyperConnectionConfig(
|
||||
hc_count=config.hc_count,
|
||||
hidden_size=config.hidden_size,
|
||||
params_dtype=torch.bfloat16,
|
||||
hc_lowrank=config.hc_lowrank,
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
hc_per_branch_norm=True,
|
||||
)
|
||||
self.hyper_connection_mixer = GatedResidual(
|
||||
hc_config,
|
||||
use_combine=False,
|
||||
prefix=maybe_prefix(prefix, "hyper_connection_mixer"),
|
||||
)
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], self.hidden_size * self.hc_count
|
||||
)
|
||||
|
||||
def _iter_qsa_attentions(self):
|
||||
"""Yield MTP attention modules that own a QSA indexer."""
|
||||
for layer in self.layers:
|
||||
attention = getattr(layer, "self_attn", None)
|
||||
if (
|
||||
attention is not None
|
||||
and getattr(attention, "indexer", None) is not None
|
||||
):
|
||||
yield attention
|
||||
|
||||
def set_skip_topk(self, skip: bool) -> None:
|
||||
"""Select on MTP step 0 and reuse its QSA indices on later steps."""
|
||||
for attention in self._iter_qsa_attentions():
|
||||
attention.indexer.skip_topk = skip
|
||||
|
||||
def compact_topk_indices(self, row_indices: torch.Tensor) -> None:
|
||||
"""Keep each request's target-aligned step-0 sparse-index row."""
|
||||
num_rows = row_indices.numel()
|
||||
for attention in self._iter_qsa_attentions():
|
||||
buffer = attention.topk_indices_buffer
|
||||
selected = buffer.index_select(0, row_indices)
|
||||
buffer[:num_rows].copy_(selected)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor | None = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
|
||||
hc_count = self.hc_count
|
||||
hidden_size = self.hidden_size
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
assert hidden_states is not None
|
||||
if inputs_embeds is None:
|
||||
assert input_ids is not None
|
||||
inputs_embeds = self.embed_input_ids(input_ids)
|
||||
# Embedding branch: pre-norm -> fc_embedding -> [T, H].
|
||||
inputs_embeds = self.pre_fc_norm_embedding(inputs_embeds)
|
||||
inputs_embeds = self.fc_embedding(inputs_embeds)
|
||||
|
||||
# Backbone hidden is multi-stream [T, hc_count*H] (scheme A:
|
||||
# the main model truly emits the pre-final-mixer multi stream
|
||||
# on the first step; subsequent steps reuse the prior draft
|
||||
# step's multi stream).
|
||||
num_tokens = hidden_states.shape[0]
|
||||
hidden_states = hidden_states.view(num_tokens, hc_count, hidden_size)
|
||||
hidden_states = self.pre_fc_norm_hidden(hidden_states.flatten(-2)).view(
|
||||
num_tokens, hc_count, hidden_size
|
||||
)
|
||||
hidden_states = self.fc_hidden(hidden_states)
|
||||
# Add the embedding residual to every branch, then fold back
|
||||
# to [T, hc_count*H] (HC outer, HS inner) for the HC decoder.
|
||||
hidden_states = inputs_embeds.unsqueeze(-2) + hidden_states
|
||||
hidden_states = hidden_states.flatten(-2)
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
full_num_tokens = positions.shape[-1]
|
||||
if self.use_hc_sequence_parallel:
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
layer = self.layers[current_step_idx]
|
||||
hidden_states, block_output, injection = layer(
|
||||
hidden_states=hidden_states,
|
||||
prev_block_output=None,
|
||||
prev_injection=None,
|
||||
positions=positions,
|
||||
input_ids=None,
|
||||
query_start_loc=None,
|
||||
ngram_context=None,
|
||||
)
|
||||
if not get_pp_group().is_last_rank:
|
||||
# As in the target model, PP carries a materialized tensor rather
|
||||
# than the delayed hidden/output/injection tuple.
|
||||
hidden_states = layer.mlp_hyper_connection.combine(
|
||||
hidden_states, block_output, injection
|
||||
)
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
|
||||
# Last PP rank finalize. Keep both:
|
||||
# (A) sample_hidden_states [T, H] -> single stream for the LM head
|
||||
# (B) multi_hidden [T, hc_count*H] -> pre-final-mixer multi stream
|
||||
# for the next draft step (zero extra compute, just kept).
|
||||
multi_hidden, sample_hidden_states, _ = (
|
||||
self.hyper_connection_mixer.combine_and_mix(
|
||||
hidden_states, block_output, injection
|
||||
)
|
||||
)
|
||||
if self.use_hc_sequence_parallel:
|
||||
# Gather both outputs together, then restore their contiguous layouts.
|
||||
packed_hidden_states = torch.cat(
|
||||
[sample_hidden_states, multi_hidden], dim=-1
|
||||
)
|
||||
packed_hidden_states = sp_all_gather(packed_hidden_states)[:full_num_tokens]
|
||||
sample_hidden_states, multi_hidden = packed_hidden_states.split(
|
||||
[hidden_size, hc_count * hidden_size], dim=-1
|
||||
)
|
||||
sample_hidden_states = sample_hidden_states.contiguous()
|
||||
multi_hidden = multi_hidden.contiguous()
|
||||
return sample_hidden_states, multi_hidden
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
weights = maybe_fuse_shared_experts(
|
||||
weights,
|
||||
enabled=self.is_fused_shared_expert_enabled,
|
||||
n_routed_experts=getattr(self.config, "num_experts", 0) or 0,
|
||||
n_shared_experts=1,
|
||||
ckpt_prefix="mlp.shared_expert",
|
||||
)
|
||||
mapper = self.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
|
||||
)
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
|
||||
)
|
||||
return loader.load_weights(weights, mapper=mapper)
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
"hidden_states": 0,
|
||||
}
|
||||
)
|
||||
class Qwen4ExpMTP(
|
||||
LocalArgmaxMixin,
|
||||
nn.Module,
|
||||
SupportsHCSP,
|
||||
SupportsPP,
|
||||
Qwen4ExpMixtureOfExperts,
|
||||
):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
"in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
|
||||
"in_proj_ba": ["in_proj_b", "in_proj_a"],
|
||||
"input_mix_weight_down_block_inject": [
|
||||
"input_mix_weight_down",
|
||||
"block_inject_weight",
|
||||
"_input_mix_padding",
|
||||
],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
self.vllm_config = vllm_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.model = Qwen4ExpMultiTokenPredictor(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "mtp"),
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
if config.tie_word_embeddings:
|
||||
self.lm_head = self.model.embed_tokens
|
||||
else:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
self.set_moe_parameters(self.model.layers)
|
||||
enable_qwen4_exp_low_latency_gemm(self, vllm_config.model_config.dtype)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor | None = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
|
||||
return self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
hidden_states,
|
||||
intermediate_tensors,
|
||||
inputs_embeds,
|
||||
spec_step_idx=spec_step_idx,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self, hidden_states: torch.Tensor, spec_step_idx: int = 0
|
||||
) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
def remap_weight_names():
|
||||
for name, weight in weights:
|
||||
remapped_name = _remap_mtp_weight_name(name)
|
||||
if remapped_name is not None:
|
||||
yield remapped_name, weight
|
||||
|
||||
mapper = WeightsMapper(
|
||||
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
|
||||
)
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
|
||||
)
|
||||
return loader.load_weights(remap_weight_names(), mapper=mapper)
|
||||
|
||||
|
||||
__all__ = ["Qwen4ExpMTP", "Qwen4ExpMultiTokenPredictor"]
|
||||
@@ -0,0 +1,3 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""AMD ROCm Qwen4Exp kernels; import leaf modules directly."""
|
||||
@@ -0,0 +1,487 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""AMD ROCm HyperConnection kernels for Qwen4Exp."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _grouped_gemma_rmsnorm_kernel(
|
||||
x_ptr,
|
||||
w_ptr,
|
||||
y_ptr,
|
||||
stride_x,
|
||||
stride_y,
|
||||
DIM: tl.constexpr,
|
||||
NUM_GROUPS: tl.constexpr,
|
||||
W_SHARED: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
) -> None:
|
||||
GROUP_DIM: tl.constexpr = DIM // NUM_GROUPS
|
||||
BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(GROUP_DIM)
|
||||
|
||||
pid = tl.program_id(0)
|
||||
group_id = pid % NUM_GROUPS
|
||||
row = pid // NUM_GROUPS
|
||||
|
||||
offs_g = tl.arange(0, BLOCK_SIZE)
|
||||
offsets = group_id * GROUP_DIM + offs_g
|
||||
mask = offs_g < GROUP_DIM
|
||||
# A [GROUP_DIM] affine is shared; a [DIM] affine follows the grouped
|
||||
# checkpoint layout.
|
||||
w_offs = offs_g if W_SHARED else offsets
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0).to(tl.float32)
|
||||
w = tl.load(w_ptr + w_offs, mask, other=0.0)
|
||||
|
||||
rrms = tl.rsqrt(tl.sum(x * x) / GROUP_DIM + EPS)
|
||||
# Gemma's (1 + w) affine is written this way to lower to an FMA.
|
||||
y = x * rrms
|
||||
y += y * w.to(tl.float32)
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(y_ptr + row * stride_y + offsets, y, mask)
|
||||
|
||||
|
||||
def _grouped_gemma_rmsnorm(
|
||||
x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
|
||||
) -> torch.Tensor:
|
||||
N, DIM = x.shape
|
||||
assert x.stride(1) == 1, "grouped Gemma RMSNorm requires unit inner stride"
|
||||
assert weight.is_contiguous(), "grouped Gemma RMSNorm weight must be contiguous"
|
||||
assert DIM % num_groups == 0
|
||||
group_dim = DIM // num_groups
|
||||
assert weight.numel() in (group_dim, DIM)
|
||||
|
||||
y = x.new_empty(x.shape)
|
||||
_grouped_gemma_rmsnorm_kernel[(N * num_groups,)](
|
||||
x,
|
||||
weight,
|
||||
y,
|
||||
x.stride(0),
|
||||
y.stride(0),
|
||||
DIM,
|
||||
num_groups,
|
||||
W_SHARED=weight.numel() == group_dim,
|
||||
EPS=eps,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_silu_kernel(
|
||||
x_ptr,
|
||||
y_ptr,
|
||||
stride_x,
|
||||
stride_y,
|
||||
DIM: tl.constexpr,
|
||||
HC: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
) -> None:
|
||||
BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(DIM)
|
||||
|
||||
row = tl.program_id(0)
|
||||
offs = tl.arange(0, BLOCK_SIZE)
|
||||
mask = offs < DIM
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
x = tl.load(x_ptr + row * stride_x + offs, mask).to(tl.float32) / HC
|
||||
y = x * tl.sigmoid(x)
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(y_ptr + row * stride_y + offs, y, mask)
|
||||
|
||||
|
||||
def _hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
num_tokens, DIM = x.shape
|
||||
assert x.stride(1) == 1
|
||||
|
||||
output = x.new_empty(x.shape)
|
||||
_hc_silu_kernel[(num_tokens,)](
|
||||
x,
|
||||
output,
|
||||
x.stride(0),
|
||||
output.stride(0),
|
||||
DIM=DIM,
|
||||
HC=hc_count,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_gate_mix_kernel(
|
||||
x_ptr,
|
||||
g_ptr,
|
||||
y_ptr,
|
||||
stride_x,
|
||||
stride_g,
|
||||
stride_y,
|
||||
DIM: tl.constexpr,
|
||||
HC: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
) -> None:
|
||||
HC_DIM: tl.constexpr = DIM // HC
|
||||
|
||||
row = tl.program_id(0)
|
||||
tile_id = tl.program_id(1)
|
||||
offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offs_inner < HC_DIM
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
# The constexpr loop is unrolled and keeps one stream live at a time.
|
||||
# Materializing [HC, BLOCK_SIZE] more than doubles latency at large M.
|
||||
acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
|
||||
for stream in tl.static_range(HC):
|
||||
offsets = stream * HC_DIM + offs_inner
|
||||
g = tl.load(g_ptr + row * stride_g + offsets, mask, other=0.0)
|
||||
x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0)
|
||||
acc += tl.sigmoid(g.to(tl.float32)) * x.to(tl.float32)
|
||||
acc /= HC
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(y_ptr + row * stride_y + offs_inner, acc, mask)
|
||||
|
||||
|
||||
def _hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
N, DIM = gate.shape
|
||||
assert x.shape == gate.shape
|
||||
assert DIM % hc_count == 0
|
||||
assert x.stride(1) == 1
|
||||
assert gate.stride(1) == 1
|
||||
|
||||
HC_DIM = DIM // hc_count
|
||||
out = x.new_empty(N, HC_DIM)
|
||||
BLOCK_SIZE = 512
|
||||
_hc_gate_mix_kernel[(N, triton.cdiv(HC_DIM, BLOCK_SIZE))](
|
||||
x,
|
||||
gate,
|
||||
out,
|
||||
x.stride(0),
|
||||
gate.stride(0),
|
||||
out.stride(0),
|
||||
DIM,
|
||||
hc_count,
|
||||
BLOCK_SIZE,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_combine_kernel(
|
||||
block_ptr,
|
||||
res_ptr,
|
||||
inj_ptr,
|
||||
out_ptr,
|
||||
stride_block,
|
||||
stride_res,
|
||||
stride_inj,
|
||||
stride_out,
|
||||
HC_DIM: tl.constexpr,
|
||||
HC: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
) -> None:
|
||||
HC_PAD: tl.constexpr = triton.next_power_of_2(HC)
|
||||
|
||||
row = tl.program_id(0)
|
||||
tile_id = tl.program_id(1)
|
||||
|
||||
offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask_inner = offs_inner < HC_DIM
|
||||
offs_hc = tl.arange(0, HC_PAD)
|
||||
mask_hc = offs_hc < HC
|
||||
offs = offs_hc[:, None] * HC_DIM + offs_inner[None, :]
|
||||
mask = mask_hc[:, None] & mask_inner[None, :]
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
|
||||
block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)
|
||||
res = tl.load(res_ptr + row * stride_res + offs, mask, other=0.0)
|
||||
|
||||
# Keeping HC as a broadcast dimension is faster here than four separate
|
||||
# residual load/store sequences.
|
||||
inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
|
||||
out = res.to(tl.float32) + block.to(tl.float32)[None, :] * inj[:, None]
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.store(out_ptr + row * stride_out + offs, out, mask=mask)
|
||||
|
||||
|
||||
def _hc_combine(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
hc_count: int,
|
||||
) -> torch.Tensor:
|
||||
N, DIM = residual.shape
|
||||
assert DIM % hc_count == 0
|
||||
hc_dim = DIM // hc_count
|
||||
assert block_output.shape == (N, hc_dim)
|
||||
assert injection_logits.shape == (N, hc_count)
|
||||
assert residual.stride(1) == 1
|
||||
assert block_output.stride(1) == 1
|
||||
assert injection_logits.stride(1) == 1
|
||||
|
||||
out = residual.new_empty(residual.shape)
|
||||
BLOCK_SIZE = 512
|
||||
_hc_combine_kernel[(N, triton.cdiv(hc_dim, BLOCK_SIZE))](
|
||||
block_output,
|
||||
residual,
|
||||
injection_logits,
|
||||
out,
|
||||
block_output.stride(0),
|
||||
residual.stride(0),
|
||||
injection_logits.stride(0),
|
||||
out.stride(0),
|
||||
hc_dim,
|
||||
hc_count,
|
||||
BLOCK_SIZE,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_combine_norm_kernel(
|
||||
block_ptr,
|
||||
res_ptr,
|
||||
inj_ptr,
|
||||
w_ptr,
|
||||
out_ptr,
|
||||
y_ptr,
|
||||
stride_block,
|
||||
stride_res,
|
||||
stride_inj,
|
||||
stride_out,
|
||||
stride_y,
|
||||
HC_DIM: tl.constexpr,
|
||||
HC: tl.constexpr,
|
||||
W_SHARED: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
) -> None:
|
||||
HC_PAD: tl.constexpr = triton.next_power_of_2(HC)
|
||||
NUM_TILES: tl.constexpr = triton.cdiv(HC_DIM, BLOCK_SIZE)
|
||||
NUM_TILES_PAD: tl.constexpr = triton.next_power_of_2(NUM_TILES)
|
||||
|
||||
row = tl.program_id(0)
|
||||
stream = tl.program_id(1)
|
||||
offs_hc = tl.arange(0, HC_PAD)
|
||||
mask_hc = offs_hc < HC
|
||||
tile_ids = tl.arange(0, NUM_TILES_PAD)
|
||||
offs_inner = tile_ids[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :]
|
||||
mask_inner = offs_inner < HC_DIM
|
||||
offs = stream * HC_DIM + offs_inner
|
||||
# Shared norm weights repeat across streams; per-branch weights use the
|
||||
# same flattened HC layout as the residual.
|
||||
w_offs = offs_inner if W_SHARED else offs
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
# Start the uncached residual load first, then issue the other combine
|
||||
# loads before consuming any of them.
|
||||
res = tl.load(res_ptr + row * stride_res + offs, mask_inner, other=0.0)
|
||||
inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
|
||||
block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)
|
||||
inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
|
||||
inj = tl.sum(tl.where(offs_hc == stream, inj, 0.0))
|
||||
# Round the materialized combine result before normalization. This matches
|
||||
# the unfused combine -> RMSNorm boundary.
|
||||
out = (res.to(tl.float32) + block.to(tl.float32) * inj).to(out_ptr.dtype.element_ty)
|
||||
tl.store(out_ptr + row * stride_out + offs, out, mask=mask_inner)
|
||||
|
||||
out = out.to(tl.float32)
|
||||
# Keep the two-axis reduction: flattening the padded tile is ~40% slower
|
||||
# at decode sizes.
|
||||
sum_sq = tl.sum(tl.sum(out * out, axis=1), axis=0)
|
||||
rrms = tl.rsqrt(sum_sq / HC_DIM + EPS)
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
# Loading the weight earlier helps decode but keeps the tile live across
|
||||
# the reduction and regresses larger batches, so defer it to the norm.
|
||||
w = tl.load(w_ptr + w_offs, mask_inner, other=0.0)
|
||||
y = out * rrms
|
||||
y += y * w.to(tl.float32)
|
||||
tl.store(y_ptr + row * stride_y + offs, y, mask_inner)
|
||||
|
||||
|
||||
def _hc_combine_norm(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_count: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
N, DIM = residual.shape
|
||||
assert DIM % hc_count == 0
|
||||
hc_dim = DIM // hc_count
|
||||
assert block_output.shape == (N, hc_dim)
|
||||
assert injection_logits.shape == (N, hc_count)
|
||||
assert residual.stride(1) == 1
|
||||
assert block_output.stride(1) == 1
|
||||
assert injection_logits.stride(1) == 1
|
||||
assert norm_weight.is_contiguous()
|
||||
assert norm_weight.numel() in (hc_dim, DIM)
|
||||
|
||||
out = residual.new_empty(residual.shape)
|
||||
y = residual.new_empty(residual.shape)
|
||||
BLOCK_SIZE = 512
|
||||
_hc_combine_norm_kernel[(N, hc_count)](
|
||||
block_output,
|
||||
residual,
|
||||
injection_logits,
|
||||
norm_weight,
|
||||
out,
|
||||
y,
|
||||
block_output.stride(0),
|
||||
residual.stride(0),
|
||||
injection_logits.stride(0),
|
||||
out.stride(0),
|
||||
y.stride(0),
|
||||
hc_dim,
|
||||
hc_count,
|
||||
W_SHARED=norm_weight.numel() == hc_dim,
|
||||
EPS=eps,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return out, y
|
||||
|
||||
|
||||
def _same_shape_fake(x: torch.Tensor, *args) -> torch.Tensor:
|
||||
return x.new_empty(x.shape)
|
||||
|
||||
|
||||
def _hc_gate_mix_fake(
|
||||
x: torch.Tensor, gate: torch.Tensor, hc_count: int
|
||||
) -> torch.Tensor:
|
||||
del gate
|
||||
return x.new_empty((x.shape[0], x.shape[1] // hc_count))
|
||||
|
||||
|
||||
def _hc_combine_fake(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
hc_count: int,
|
||||
) -> torch.Tensor:
|
||||
del block_output, injection_logits, hc_count
|
||||
return residual.new_empty(residual.shape)
|
||||
|
||||
|
||||
def _hc_combine_norm_fake(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_count: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
del block_output, injection_logits, norm_weight, eps, hc_count
|
||||
return residual.new_empty(residual.shape), residual.new_empty(residual.shape)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_grouped_gemma_rmsnorm",
|
||||
op_func=_grouped_gemma_rmsnorm,
|
||||
fake_impl=_same_shape_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_hc_silu",
|
||||
op_func=_hc_silu,
|
||||
fake_impl=_same_shape_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_hc_gate_mix",
|
||||
op_func=_hc_gate_mix,
|
||||
fake_impl=_hc_gate_mix_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_hc_combine",
|
||||
op_func=_hc_combine,
|
||||
fake_impl=_hc_combine_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_hc_combine_norm",
|
||||
op_func=_hc_combine_norm,
|
||||
fake_impl=_hc_combine_norm_fake,
|
||||
)
|
||||
|
||||
|
||||
def grouped_gemma_rmsnorm(
|
||||
x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.qwen4_exp_grouped_gemma_rmsnorm(x, weight, eps, num_groups)
|
||||
|
||||
|
||||
def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
return torch.ops.vllm.qwen4_exp_hc_silu(x, hc_count)
|
||||
|
||||
|
||||
def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
|
||||
return torch.ops.vllm.qwen4_exp_hc_gate_mix(x, gate, hc_count)
|
||||
|
||||
|
||||
def hc_combine(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
hc_count: int,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.qwen4_exp_hc_combine(
|
||||
residual, block_output, injection_logits, hc_count
|
||||
)
|
||||
|
||||
|
||||
def hc_combine_norm(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_count: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.ops.vllm.qwen4_exp_hc_combine_norm(
|
||||
residual,
|
||||
block_output,
|
||||
injection_logits,
|
||||
norm_weight,
|
||||
eps,
|
||||
hc_count,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"grouped_gemma_rmsnorm",
|
||||
"hc_combine",
|
||||
"hc_combine_norm",
|
||||
"hc_gate_mix",
|
||||
"hc_silu",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,703 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused QSA prepare kernel for the AMD Qwen4Exp path.
|
||||
|
||||
Started as a copy of the NVIDIA kernel (``nvidia/ops/qsa_prepare.py``) and is
|
||||
kept separate so either vendor can change its own copy without re-validating
|
||||
the other. The state source select is two masked loads rather than a pointer
|
||||
select, which ROCm Triton rejects.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
def supports_fused_pre_indexer(
|
||||
rotary_emb: nn.Module,
|
||||
head_dim: int,
|
||||
num_kv_heads: int,
|
||||
compress_ratio: int,
|
||||
) -> bool:
|
||||
"""Report whether this indexer's shapes match the fused kernel's assumptions.
|
||||
|
||||
The kernel hard-codes the rotary layout and the single-KV-head group
|
||||
compression it was written for; everything it rejects has a working unfused
|
||||
path.
|
||||
"""
|
||||
rotary_dim = int(rotary_emb.rotary_dim)
|
||||
mrope_section = getattr(rotary_emb, "mrope_section", None)
|
||||
return (
|
||||
bool(getattr(rotary_emb, "is_neox_style", False))
|
||||
and (
|
||||
not mrope_section
|
||||
or (
|
||||
len(mrope_section) == 3
|
||||
and sum(mrope_section) == rotary_dim // 2
|
||||
and bool(getattr(rotary_emb, "mrope_interleaved", False))
|
||||
)
|
||||
)
|
||||
and head_dim == 128
|
||||
and rotary_dim == 64
|
||||
and num_kv_heads == 1
|
||||
and compress_ratio > 1
|
||||
and compress_ratio & (compress_ratio - 1) == 0
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _norm_rope(
|
||||
x,
|
||||
pos_t,
|
||||
pos_h,
|
||||
pos_w,
|
||||
cos_sin_ptr,
|
||||
cos_sin_stride,
|
||||
norm_weight_ptr,
|
||||
eps,
|
||||
IS_MROPE: tl.constexpr,
|
||||
MROPE_H: tl.constexpr,
|
||||
MROPE_W: tl.constexpr,
|
||||
):
|
||||
"""Apply Gemma RMSNorm and selected-axis NeoX RoPE to register rows."""
|
||||
TILE_T: tl.constexpr = x.shape[0]
|
||||
TILE_H: tl.constexpr = x.shape[1]
|
||||
D: tl.constexpr = x.shape[2]
|
||||
ROWS: tl.constexpr = TILE_T * TILE_H
|
||||
HALF: tl.constexpr = D // 2
|
||||
QUARTER: tl.constexpr = D // 4
|
||||
pairs = tl.arange(0, QUARTER)
|
||||
if IS_MROPE:
|
||||
# Qwen interleaves temporal, height, and width rotary pairs. Each axis
|
||||
# still indexes the same position-major cos/sin table.
|
||||
h_mask = ((pairs % 3) == 1) & (pairs <= 3 * MROPE_H)
|
||||
w_mask = ((pairs % 3) == 2) & (pairs <= 3 * MROPE_W)
|
||||
t_mask = ~(h_mask | w_mask)
|
||||
base = cos_sin_ptr + pairs[None, :]
|
||||
pos_rows = (pos_t, pos_h, pos_w)
|
||||
axis_masks = (t_mask, h_mask, w_mask)
|
||||
cos = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty)
|
||||
sin = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty)
|
||||
for axis in tl.static_range(3):
|
||||
cos += tl.load(
|
||||
base + pos_rows[axis][:, None] * cos_sin_stride,
|
||||
mask=axis_masks[axis][None, :],
|
||||
other=0,
|
||||
)
|
||||
sin += tl.load(
|
||||
base + pos_rows[axis][:, None] * cos_sin_stride + QUARTER,
|
||||
mask=axis_masks[axis][None, :],
|
||||
other=0,
|
||||
)
|
||||
else:
|
||||
cos = tl.load(cos_sin_ptr + pos_t[:, None] * cos_sin_stride + pairs[None, :])
|
||||
sin = tl.load(
|
||||
cos_sin_ptr + pos_t[:, None] * cos_sin_stride + QUARTER + pairs[None, :]
|
||||
)
|
||||
|
||||
cos = tl.reshape(
|
||||
tl.broadcast_to(cos[:, None, :], (TILE_T, TILE_H, QUARTER)),
|
||||
(ROWS, QUARTER),
|
||||
)
|
||||
sin = tl.reshape(
|
||||
tl.broadcast_to(sin[:, None, :], (TILE_T, TILE_H, QUARTER)),
|
||||
(ROWS, QUARTER),
|
||||
)
|
||||
x = tl.reshape(x, (ROWS, D)).to(tl.float32)
|
||||
weight = tl.load(norm_weight_ptr + tl.arange(0, D)).to(tl.float32) + 1.0
|
||||
rrms = tl.rsqrt(tl.sum(x * x, axis=1) / D + eps)
|
||||
y = (x * rrms[:, None] * weight[None, :]).to(cos.dtype)
|
||||
rotated, passthrough = tl.split(
|
||||
tl.permute(tl.reshape(y, (ROWS, 2, HALF)), (0, 2, 1))
|
||||
)
|
||||
r0, r1 = tl.split(tl.permute(tl.reshape(rotated, (ROWS, 2, QUARTER)), (0, 2, 1)))
|
||||
out0 = r0 * cos - r1 * sin
|
||||
out1 = r1 * cos + r0 * sin
|
||||
rotated = tl.reshape(tl.permute(tl.join(out0, out1), (0, 2, 1)), (ROWS, HALF))
|
||||
result = tl.reshape(tl.permute(tl.join(rotated, passthrough), (0, 2, 1)), (ROWS, D))
|
||||
return tl.reshape(result, (TILE_T, TILE_H, D))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _to_dst_dtype(x, dst, scale):
|
||||
"""Round to BF16 like the unfused path, then scale for an FP8 destination."""
|
||||
out_ty = dst.dtype.element_ty
|
||||
x = x.to(tl.bfloat16)
|
||||
if out_ty == tl.float8e4nv:
|
||||
x = x.to(tl.float32) / scale
|
||||
return x.to(out_ty)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_rotated(dst, y, o1, o2, scale):
|
||||
"""Store a normalized head whose first ``2 * len(o1)`` dims are rotated."""
|
||||
HALF: tl.constexpr = o1.shape[0]
|
||||
dims = tl.arange(0, y.shape[0])
|
||||
rot = tl.arange(0, HALF)
|
||||
tl.store(dst + dims, _to_dst_dtype(y, dst, scale), mask=dims >= 2 * HALF)
|
||||
tl.store(dst + rot, _to_dst_dtype(o1, dst, scale))
|
||||
tl.store(dst + HALF + rot, _to_dst_dtype(o2, dst, scale))
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"num_tokens",
|
||||
"num_state_blocks",
|
||||
"num_compressed_blocks",
|
||||
"num_k_work",
|
||||
]
|
||||
)
|
||||
def _qsa_prepare_kernel(
|
||||
q_ptr,
|
||||
q_stride_token,
|
||||
k_ptr,
|
||||
k_stride_token,
|
||||
pos_ptr,
|
||||
pos_stride_axis,
|
||||
pos_stride_token,
|
||||
cos_sin_ptr,
|
||||
q_norm_weight_ptr,
|
||||
k_norm_weight_ptr,
|
||||
eps,
|
||||
q_out_ptr,
|
||||
q_out_stride_token,
|
||||
q_out_stride_head,
|
||||
state_cache_ptr,
|
||||
state_cache_stride_block,
|
||||
state_cache_stride_token,
|
||||
state_slots_ptr,
|
||||
state_table_ptr,
|
||||
state_table_stride_req,
|
||||
query_start_loc_ptr,
|
||||
logical_positions_ptr,
|
||||
compressed_slots_ptr,
|
||||
k_work_metadata_ptr,
|
||||
compressed_cache_ptr,
|
||||
compressed_cache_stride_block,
|
||||
compressed_cache_stride_token,
|
||||
num_tokens,
|
||||
num_state_blocks,
|
||||
num_compressed_blocks,
|
||||
num_k_work,
|
||||
HQ: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
TILE_T_Q: tl.constexpr,
|
||||
TILE_H_Q: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
STATE_SIZE: tl.constexpr,
|
||||
COMP_PAGE_SIZE: tl.constexpr,
|
||||
IS_2D_POSITIONS: tl.constexpr,
|
||||
IS_K_MROPE: tl.constexpr,
|
||||
CACHE_HAS_ROPE_POS: tl.constexpr,
|
||||
MROPE_H: tl.constexpr,
|
||||
MROPE_W: tl.constexpr,
|
||||
main_qkv_ptr,
|
||||
main_q_norm_weight_ptr,
|
||||
main_k_norm_weight_ptr,
|
||||
main_eps,
|
||||
main_q_out_ptr,
|
||||
main_gate_out_ptr,
|
||||
main_cache_ptr,
|
||||
main_cache_stride_block,
|
||||
main_cache_stride_token,
|
||||
main_cache_stride_head,
|
||||
main_slots_ptr,
|
||||
main_k_scale,
|
||||
main_v_scale,
|
||||
MAIN_HQ: tl.constexpr,
|
||||
MAIN_HK: tl.constexpr,
|
||||
MAIN_D: tl.constexpr,
|
||||
MAIN_PAGE_SIZE: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
num_index_work = num_k_work + tl.cdiv(num_tokens, TILE_T_Q) * tl.cdiv(HQ, TILE_H_Q)
|
||||
if pid >= num_index_work:
|
||||
# Main attention: one program per (token, Q or KV head) after the
|
||||
# indexer work. RoPE covers the first D // 2 dims, the width of the
|
||||
# shared cos/sin table.
|
||||
main_pid = pid - num_index_work
|
||||
token = (main_pid // (MAIN_HQ + MAIN_HK)).to(tl.int64)
|
||||
head = main_pid % (MAIN_HQ + MAIN_HK)
|
||||
HALF: tl.constexpr = D // 4
|
||||
dims = tl.arange(0, MAIN_D)
|
||||
rot = tl.arange(0, HALF)
|
||||
row = main_qkv_ptr + token * (2 * (MAIN_HQ + MAIN_HK) * MAIN_D)
|
||||
is_k = head >= MAIN_HQ
|
||||
kv_head = head - MAIN_HQ
|
||||
if is_k:
|
||||
src = row + (2 * MAIN_HQ + kv_head) * MAIN_D
|
||||
weight_ptr = main_k_norm_weight_ptr
|
||||
else:
|
||||
src = row + 2 * head * MAIN_D
|
||||
weight_ptr = main_q_norm_weight_ptr
|
||||
x = tl.load(src + dims).to(tl.float32)
|
||||
inv_rms = tl.rsqrt(tl.sum(x * x, axis=0) / MAIN_D + main_eps)
|
||||
w = tl.load(weight_ptr + dims).to(tl.float32) + 1.0
|
||||
y = x * inv_rms * w
|
||||
x1 = tl.load(src + rot).to(tl.float32)
|
||||
x2 = tl.load(src + HALF + rot).to(tl.float32)
|
||||
w1 = tl.load(weight_ptr + rot).to(tl.float32) + 1.0
|
||||
w2 = tl.load(weight_ptr + HALF + rot).to(tl.float32) + 1.0
|
||||
x1 = (x1 * inv_rms * w1).to(tl.bfloat16).to(tl.float32)
|
||||
x2 = (x2 * inv_rms * w2).to(tl.bfloat16).to(tl.float32)
|
||||
pos = tl.load(pos_ptr + token * pos_stride_token).to(tl.int64)
|
||||
if IS_2D_POSITIONS:
|
||||
pos_h = tl.load(pos_ptr + pos_stride_axis + token * pos_stride_token)
|
||||
pos_w = tl.load(pos_ptr + 2 * pos_stride_axis + token * pos_stride_token)
|
||||
is_h = (rot % 3 == 1) & (rot < 3 * MROPE_H)
|
||||
is_w = (rot % 3 == 2) & (rot < 3 * MROPE_W)
|
||||
pos = tl.where(
|
||||
is_h, pos_h.to(tl.int64), tl.where(is_w, pos_w.to(tl.int64), pos)
|
||||
)
|
||||
cos = tl.load(cos_sin_ptr + pos * (D // 2) + rot).to(tl.float32)
|
||||
sin = tl.load(cos_sin_ptr + pos * (D // 2) + HALF + rot).to(tl.float32)
|
||||
o1 = x1 * cos - x2 * sin
|
||||
o2 = x2 * cos + x1 * sin
|
||||
if is_k:
|
||||
slot = tl.load(main_slots_ptr + token).to(tl.int64)
|
||||
if slot >= 0:
|
||||
dst = (
|
||||
main_cache_ptr
|
||||
+ (slot // MAIN_PAGE_SIZE) * main_cache_stride_block
|
||||
+ (slot % MAIN_PAGE_SIZE) * main_cache_stride_token
|
||||
+ kv_head * main_cache_stride_head
|
||||
)
|
||||
_store_rotated(dst, y, o1, o2, main_k_scale)
|
||||
v = tl.load(src + MAIN_HK * MAIN_D + dims)
|
||||
tl.store(dst + MAIN_D + dims, _to_dst_dtype(v, dst, main_v_scale))
|
||||
else:
|
||||
out = (token * MAIN_HQ + head) * MAIN_D
|
||||
_store_rotated(main_q_out_ptr + out, y, o1, o2, None)
|
||||
gate = tl.load(src + MAIN_D + dims)
|
||||
tl.store(main_gate_out_ptr + out + dims, gate)
|
||||
return
|
||||
# K work occupies the first programs; the remaining programs tile Q. This
|
||||
# keeps both paths in one launch while leaving their register shapes
|
||||
# independent.
|
||||
if pid >= num_k_work:
|
||||
q_pid = pid - num_k_work
|
||||
num_head_tiles: tl.constexpr = tl.cdiv(HQ, TILE_H_Q)
|
||||
token_tile = q_pid // num_head_tiles
|
||||
head_tile = q_pid % num_head_tiles
|
||||
tokens = (token_tile * TILE_T_Q + tl.arange(0, TILE_T_Q)).to(tl.int64)
|
||||
heads = head_tile * TILE_H_Q + tl.arange(0, TILE_H_Q)
|
||||
valid_tokens = tokens < num_tokens
|
||||
valid_heads = heads < HQ
|
||||
dims = tl.arange(0, D)
|
||||
mask = valid_tokens[:, None, None] & valid_heads[None, :, None]
|
||||
x = tl.load(
|
||||
q_ptr
|
||||
+ tokens[:, None, None] * q_stride_token
|
||||
+ heads[None, :, None] * D
|
||||
+ dims[None, None, :],
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
)
|
||||
pos_t = tl.load(pos_ptr + tokens * pos_stride_token, mask=valid_tokens, other=0)
|
||||
if IS_2D_POSITIONS:
|
||||
pos_h = tl.load(
|
||||
pos_ptr + pos_stride_axis + tokens * pos_stride_token,
|
||||
mask=valid_tokens,
|
||||
other=0,
|
||||
)
|
||||
pos_w = tl.load(
|
||||
pos_ptr + 2 * pos_stride_axis + tokens * pos_stride_token,
|
||||
mask=valid_tokens,
|
||||
other=0,
|
||||
)
|
||||
else:
|
||||
pos_h = pos_t
|
||||
pos_w = pos_t
|
||||
y = _norm_rope(
|
||||
x,
|
||||
pos_t,
|
||||
pos_h,
|
||||
pos_w,
|
||||
cos_sin_ptr,
|
||||
D // 2,
|
||||
q_norm_weight_ptr,
|
||||
eps,
|
||||
IS_2D_POSITIONS,
|
||||
MROPE_H,
|
||||
MROPE_W,
|
||||
)
|
||||
tl.store(
|
||||
q_out_ptr
|
||||
+ tokens[:, None, None] * q_out_stride_token
|
||||
+ heads[None, :, None] * q_out_stride_head
|
||||
+ dims[None, None, :],
|
||||
y,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
if pid < num_k_work:
|
||||
# One work item owns one completed compression group. Work item zero
|
||||
# additionally commits the request's current raw-K suffix below.
|
||||
work_metadata = tl.load(k_work_metadata_ptr + pid * 2 + tl.arange(0, 2))
|
||||
request, work_in_request = tl.split(work_metadata)
|
||||
if request < 0:
|
||||
return
|
||||
|
||||
query_start = tl.load(query_start_loc_ptr + request)
|
||||
query_end = tl.load(query_start_loc_ptr + request + 1)
|
||||
query_len = query_end - query_start
|
||||
chunk_end = tl.load(logical_positions_ptr + query_end - 1)
|
||||
chunk_start = chunk_end - query_len + 1
|
||||
num_groups = (chunk_end + 1) // COMPRESS_RATIO - chunk_start // COMPRESS_RATIO
|
||||
dims = tl.arange(0, D)
|
||||
|
||||
if work_in_request < num_groups:
|
||||
first_boundary = (
|
||||
(chunk_start + COMPRESS_RATIO) // COMPRESS_RATIO
|
||||
) * COMPRESS_RATIO - 1
|
||||
end_position = first_boundary + work_in_request * COMPRESS_RATIO
|
||||
boundary_token = query_start + end_position - chunk_start
|
||||
valid_token = (
|
||||
(boundary_token >= query_start)
|
||||
& (boundary_token < query_end)
|
||||
& (boundary_token < num_tokens)
|
||||
)
|
||||
compressed_slot = tl.load(
|
||||
compressed_slots_ptr + boundary_token,
|
||||
mask=valid_token,
|
||||
other=-1,
|
||||
)
|
||||
valid = (
|
||||
valid_token
|
||||
& (compressed_slot >= 0)
|
||||
& (compressed_slot < num_compressed_blocks * COMP_PAGE_SIZE)
|
||||
)
|
||||
state_block = tl.load(state_table_ptr + request * state_table_stride_req)
|
||||
state_block_valid = (state_block >= 0) & (state_block < num_state_blocks)
|
||||
safe_state_block = tl.maximum(state_block, 0).to(tl.int64)
|
||||
group_offsets = tl.arange(0, COMPRESS_RATIO)
|
||||
source_positions = end_position - (COMPRESS_RATIO - 1) + group_offsets
|
||||
source_in_chunk = source_positions >= chunk_start
|
||||
source_in_cache = source_positions < chunk_start
|
||||
source_tokens = query_start + source_positions - chunk_start
|
||||
source_tokens_valid = (
|
||||
(source_tokens >= query_start)
|
||||
& (source_tokens < query_end)
|
||||
& (source_tokens < num_tokens)
|
||||
)
|
||||
current_base = (
|
||||
k_ptr
|
||||
+ tl.maximum(source_tokens, 0).to(tl.int64)[:, None] * k_stride_token
|
||||
)
|
||||
cached_base = (
|
||||
state_cache_ptr
|
||||
+ safe_state_block * state_cache_stride_block
|
||||
+ (source_positions % STATE_SIZE)[:, None] * state_cache_stride_token
|
||||
)
|
||||
# Only the first completed group can cross the chunk boundary, so
|
||||
# every row comes from exactly one of the two sources. Selecting
|
||||
# between the base pointers would need a pointer-typed tl.where,
|
||||
# which ROCm's Triton rejects ("'tt.addptr' op failed to verify
|
||||
# that result type matches ptr type"), so select the loaded values
|
||||
# instead. The two masks are complementary, so no lane fetches from
|
||||
# both and memory traffic is unchanged.
|
||||
#
|
||||
# Both bases still need the alignment stated explicitly. Without it
|
||||
# Triton vectorizes these loads differently, which reorders the
|
||||
# pooling reduction below; float addition is not associative, so the
|
||||
# compressed rows stop matching the unfused path bit for bit and an
|
||||
# FP8 cache rounds a near-zero pooled value to the wrong code.
|
||||
current_base = tl.multiple_of(current_base, (8, 8))
|
||||
cached_base = tl.multiple_of(cached_base, (8, 8))
|
||||
from_chunk = valid & source_in_chunk[:, None] & source_tokens_valid[:, None]
|
||||
from_cache = valid & source_in_cache[:, None] & state_block_valid
|
||||
current = tl.load(current_base + dims[None, :], mask=from_chunk, other=0.0)
|
||||
cached = tl.load(cached_base + dims[None, :], mask=from_cache, other=0.0)
|
||||
source = tl.where(source_in_chunk[:, None], current, cached).to(tl.float32)
|
||||
# Match the unfused path's BF16 pooled tensor before RMSNorm.
|
||||
pooled = (
|
||||
(tl.sum(source, axis=0) / COMPRESS_RATIO).to(tl.bfloat16).to(tl.float32)
|
||||
)
|
||||
|
||||
first_position = end_position - (COMPRESS_RATIO - 1)
|
||||
if CACHE_HAS_ROPE_POS:
|
||||
# RoPE uses the first token in the pooled group. Its exact MRoPE
|
||||
# coordinates may live in this chunk or the raw-state ring.
|
||||
first_in_chunk = first_position >= chunk_start
|
||||
first_token = query_start + first_position - chunk_start
|
||||
first_token_valid = (
|
||||
(first_token >= query_start)
|
||||
& (first_token < query_end)
|
||||
& (first_token < num_tokens)
|
||||
)
|
||||
safe_first_token = tl.maximum(first_token, 0)
|
||||
load_current_position = first_in_chunk & first_token_valid
|
||||
first_pos_t = tl.load(
|
||||
pos_ptr + safe_first_token * pos_stride_token,
|
||||
mask=load_current_position,
|
||||
other=0,
|
||||
)
|
||||
if IS_2D_POSITIONS:
|
||||
first_pos_h = tl.load(
|
||||
pos_ptr + pos_stride_axis + safe_first_token * pos_stride_token,
|
||||
mask=load_current_position,
|
||||
other=0,
|
||||
)
|
||||
first_pos_w = tl.load(
|
||||
pos_ptr
|
||||
+ 2 * pos_stride_axis
|
||||
+ safe_first_token * pos_stride_token,
|
||||
mask=load_current_position,
|
||||
other=0,
|
||||
)
|
||||
else:
|
||||
first_pos_h = first_pos_t
|
||||
first_pos_w = first_pos_t
|
||||
tail = (
|
||||
state_cache_ptr
|
||||
+ safe_state_block * state_cache_stride_block
|
||||
+ (first_position % STATE_SIZE) * state_cache_stride_token
|
||||
+ D
|
||||
).to(tl.pointer_type(tl.int64))
|
||||
load_cached_position = ~first_in_chunk & state_block_valid
|
||||
cached_pos_t = tl.load(tail, mask=load_cached_position, other=0)
|
||||
cached_pos_h = tl.load(tail + 1, mask=load_cached_position, other=0)
|
||||
cached_pos_w = tl.load(tail + 2, mask=load_cached_position, other=0)
|
||||
pos_t = tl.where(first_in_chunk, first_pos_t.to(tl.int64), cached_pos_t)
|
||||
pos_h = tl.where(first_in_chunk, first_pos_h.to(tl.int64), cached_pos_h)
|
||||
pos_w = tl.where(first_in_chunk, first_pos_w.to(tl.int64), cached_pos_w)
|
||||
else:
|
||||
pos_t = first_position
|
||||
pos_h = first_position
|
||||
pos_w = first_position
|
||||
y = _norm_rope(
|
||||
tl.reshape(pooled, (1, 1, D)),
|
||||
pos_t + tl.arange(0, 1),
|
||||
pos_h + tl.arange(0, 1),
|
||||
pos_w + tl.arange(0, 1),
|
||||
cos_sin_ptr,
|
||||
D // 2,
|
||||
k_norm_weight_ptr,
|
||||
eps,
|
||||
IS_K_MROPE,
|
||||
MROPE_H,
|
||||
MROPE_W,
|
||||
)
|
||||
compressed_block = (compressed_slot // COMP_PAGE_SIZE).to(tl.int64)
|
||||
compressed_row = compressed_slot % COMP_PAGE_SIZE
|
||||
tl.store(
|
||||
compressed_cache_ptr
|
||||
+ compressed_block * compressed_cache_stride_block
|
||||
+ compressed_row * compressed_cache_stride_token
|
||||
+ dims,
|
||||
tl.reshape(y, (D,)),
|
||||
mask=valid,
|
||||
)
|
||||
|
||||
if work_in_request == 0:
|
||||
# This CTA may have just read historical rows from the circular buffer.
|
||||
# Keep every lane past those loads before overwriting the same ring.
|
||||
tl.debug_barrier()
|
||||
# One CTA per request commits only the suffix retained by the ring.
|
||||
num_state_rows = tl.minimum(query_len, STATE_SIZE)
|
||||
for state_offset in tl.range(0, num_state_rows):
|
||||
token = (query_end - num_state_rows + state_offset).to(tl.int64)
|
||||
valid_token = (
|
||||
(token >= query_start) & (token < query_end) & (token < num_tokens)
|
||||
)
|
||||
slot = tl.load(state_slots_ptr + token, mask=valid_token, other=-1)
|
||||
valid_slot = (
|
||||
valid_token & (slot >= 0) & (slot < num_state_blocks * STATE_SIZE)
|
||||
)
|
||||
safe_slot = tl.maximum(slot, 0)
|
||||
state_row = (
|
||||
state_cache_ptr
|
||||
+ (safe_slot // STATE_SIZE).to(tl.int64) * state_cache_stride_block
|
||||
+ (safe_slot % STATE_SIZE) * state_cache_stride_token
|
||||
)
|
||||
k = tl.load(
|
||||
k_ptr + tl.maximum(token, 0) * k_stride_token + dims,
|
||||
mask=valid_slot,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(state_row + dims, k, mask=valid_slot)
|
||||
if CACHE_HAS_ROPE_POS:
|
||||
pos_t = tl.load(
|
||||
pos_ptr + tl.maximum(token, 0) * pos_stride_token,
|
||||
mask=valid_slot,
|
||||
other=0,
|
||||
)
|
||||
if IS_2D_POSITIONS:
|
||||
pos_h = tl.load(
|
||||
pos_ptr
|
||||
+ pos_stride_axis
|
||||
+ tl.maximum(token, 0) * pos_stride_token,
|
||||
mask=valid_slot,
|
||||
other=0,
|
||||
)
|
||||
pos_w = tl.load(
|
||||
pos_ptr
|
||||
+ 2 * pos_stride_axis
|
||||
+ tl.maximum(token, 0) * pos_stride_token,
|
||||
mask=valid_slot,
|
||||
other=0,
|
||||
)
|
||||
else:
|
||||
pos_h = pos_t
|
||||
pos_w = pos_t
|
||||
tail = (state_row + D).to(tl.pointer_type(tl.int64))
|
||||
tl.store(tail, pos_t.to(tl.int64), mask=valid_slot)
|
||||
tl.store(tail + 1, pos_h.to(tl.int64), mask=valid_slot)
|
||||
tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot)
|
||||
|
||||
|
||||
def qsa_prepare(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
q_norm_weight: torch.Tensor,
|
||||
k_norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
q_out: torch.Tensor,
|
||||
state_cache: torch.Tensor,
|
||||
state_slots: torch.Tensor,
|
||||
state_block_table: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
logical_positions: torch.Tensor,
|
||||
compressed_cache: torch.Tensor,
|
||||
compressed_slots: torch.Tensor,
|
||||
k_work_metadata: torch.Tensor,
|
||||
*,
|
||||
compress_ratio: int,
|
||||
mrope_section: tuple[int, int, int] | None,
|
||||
rope_pos_offset: int | None,
|
||||
main_qkv: torch.Tensor,
|
||||
main_q_norm_weight: torch.Tensor,
|
||||
main_k_norm_weight: torch.Tensor,
|
||||
main_eps: float,
|
||||
main_kv_cache: torch.Tensor,
|
||||
main_slot_mapping: torch.Tensor,
|
||||
main_k_scale: float,
|
||||
main_v_scale: float,
|
||||
main_gate_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Normalize Q, compress K, then update the circular raw state.
|
||||
|
||||
Also prepares the main attention (QK-norm/RoPE, gate copy, K/V cache write)
|
||||
and returns its Q and gate. ``main_gate_out`` lets the gate land in a
|
||||
caller-owned buffer, e.g. one that outlives an opaque custom op.
|
||||
"""
|
||||
num_tokens = q.shape[0]
|
||||
main_head_dim = main_kv_cache.shape[-1] // 2
|
||||
num_main_kv_heads = main_kv_cache.shape[2]
|
||||
num_main_q_heads = main_qkv.shape[1] // (2 * main_head_dim) - num_main_kv_heads
|
||||
main_q_out = main_qkv.new_empty(num_tokens, num_main_q_heads, main_head_dim)
|
||||
if main_gate_out is None:
|
||||
main_gate_out = torch.empty_like(main_q_out)
|
||||
else:
|
||||
assert main_gate_out.shape == main_q_out.shape
|
||||
assert main_gate_out.is_contiguous()
|
||||
if num_tokens == 0:
|
||||
return main_q_out, main_gate_out
|
||||
num_q_heads, head_dim = q_out.shape[1:]
|
||||
assert cos_sin_cache.shape[-1] * 2 == head_dim
|
||||
assert q.shape == (num_tokens, num_q_heads * head_dim)
|
||||
assert k.shape == (num_tokens, head_dim)
|
||||
assert q.stride(-1) == 1
|
||||
assert k.stride(-1) == 1
|
||||
assert q_out.stride(-1) == 1
|
||||
assert cos_sin_cache.is_contiguous()
|
||||
assert state_cache.stride(-1) == 1
|
||||
assert compressed_cache.stride(-1) == 1
|
||||
assert k_work_metadata.ndim == 2 and k_work_metadata.shape[1] == 2
|
||||
is_2d_positions = positions.ndim == 2
|
||||
is_k_mrope = bool(mrope_section)
|
||||
cache_has_rope_pos = rope_pos_offset is not None
|
||||
assert rope_pos_offset is None or rope_pos_offset == head_dim
|
||||
if is_2d_positions:
|
||||
assert positions.shape == (3, num_tokens)
|
||||
assert is_k_mrope
|
||||
pos_stride_axis, pos_stride_token = positions.stride()
|
||||
else:
|
||||
assert positions.shape == (num_tokens,)
|
||||
pos_stride_axis, pos_stride_token = 0, positions.stride(0)
|
||||
section = mrope_section if mrope_section is not None else (0, 0, 0)
|
||||
assert len(section) == 3
|
||||
qkv_width = 2 * (num_main_q_heads + num_main_kv_heads) * main_head_dim
|
||||
assert main_qkv.shape == (num_tokens, qkv_width) and main_qkv.is_contiguous()
|
||||
assert main_slot_mapping.shape == (num_tokens,)
|
||||
|
||||
# Swept on MI355X (64-lane wavefronts) over TILE_T_Q 1-16, TILE_H_Q 1-4 and
|
||||
# num_warps 1/2/4, from 4 to 16384 tokens: no config beat these by more
|
||||
# than noise, and 2 or 4 warps were slower everywhere.
|
||||
if num_tokens <= 4096:
|
||||
TILE_T_Q, TILE_H_Q = 2, 2
|
||||
else:
|
||||
TILE_T_Q, TILE_H_Q = 2, 4
|
||||
num_k_work = k_work_metadata.shape[0]
|
||||
num_q_work = triton.cdiv(num_tokens, TILE_T_Q) * triton.cdiv(num_q_heads, TILE_H_Q)
|
||||
num_main_work = num_tokens * (num_main_q_heads + num_main_kv_heads)
|
||||
_qsa_prepare_kernel[(num_k_work + num_q_work + num_main_work,)](
|
||||
q,
|
||||
q.stride(0),
|
||||
k,
|
||||
k.stride(0),
|
||||
positions,
|
||||
pos_stride_axis,
|
||||
pos_stride_token,
|
||||
cos_sin_cache,
|
||||
q_norm_weight,
|
||||
k_norm_weight,
|
||||
eps,
|
||||
q_out,
|
||||
q_out.stride(0),
|
||||
q_out.stride(1),
|
||||
state_cache,
|
||||
state_cache.stride(0),
|
||||
state_cache.stride(1),
|
||||
state_slots,
|
||||
state_block_table,
|
||||
state_block_table.stride(0),
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
compressed_slots,
|
||||
k_work_metadata,
|
||||
compressed_cache,
|
||||
compressed_cache.stride(0),
|
||||
compressed_cache.stride(1),
|
||||
num_tokens,
|
||||
state_cache.shape[0],
|
||||
compressed_cache.shape[0],
|
||||
num_k_work,
|
||||
HQ=num_q_heads,
|
||||
D=head_dim,
|
||||
TILE_T_Q=TILE_T_Q,
|
||||
TILE_H_Q=TILE_H_Q,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
STATE_SIZE=state_cache.shape[1],
|
||||
COMP_PAGE_SIZE=compressed_cache.shape[1],
|
||||
IS_2D_POSITIONS=is_2d_positions,
|
||||
IS_K_MROPE=is_k_mrope,
|
||||
CACHE_HAS_ROPE_POS=cache_has_rope_pos,
|
||||
MROPE_H=section[1],
|
||||
MROPE_W=section[2],
|
||||
main_qkv_ptr=main_qkv,
|
||||
main_q_norm_weight_ptr=main_q_norm_weight,
|
||||
main_k_norm_weight_ptr=main_k_norm_weight,
|
||||
main_eps=main_eps,
|
||||
main_q_out_ptr=main_q_out,
|
||||
main_gate_out_ptr=main_gate_out,
|
||||
main_cache_ptr=main_kv_cache,
|
||||
main_cache_stride_block=main_kv_cache.stride(0),
|
||||
main_cache_stride_token=main_kv_cache.stride(1),
|
||||
main_cache_stride_head=main_kv_cache.stride(2),
|
||||
main_slots_ptr=main_slot_mapping,
|
||||
main_k_scale=main_k_scale,
|
||||
main_v_scale=main_v_scale,
|
||||
MAIN_HQ=num_main_q_heads,
|
||||
MAIN_HK=num_main_kv_heads,
|
||||
MAIN_D=main_head_dim,
|
||||
MAIN_PAGE_SIZE=main_kv_cache.shape[1],
|
||||
num_warps=1,
|
||||
)
|
||||
return main_q_out, main_gate_out
|
||||
|
||||
|
||||
__all__ = ["qsa_prepare", "supports_fused_pre_indexer"]
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.models.common.ops import sequence_parallel as common_sp
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
def _sp_shard_op(x: torch.Tensor) -> torch.Tensor:
|
||||
output = common_sp.sp_shard(x)
|
||||
# A functional custom op must not return a view of its input.
|
||||
unpadded = x.shape[0] % get_tensor_model_parallel_world_size() == 0
|
||||
return output.clone() if unpadded else output
|
||||
|
||||
|
||||
def _sp_local_fake(x: torch.Tensor) -> torch.Tensor:
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
return x.new_empty((cdiv(x.shape[0], tp_size), *x.shape[1:]))
|
||||
|
||||
|
||||
# Dynamo would specialize the padding branch on the traced token count, and
|
||||
# vLLM drops the resulting guards. Keep the branch opaque when compiling.
|
||||
direct_register_custom_op(
|
||||
op_name="sp_reduce_scatter",
|
||||
op_func=common_sp.sp_reduce_scatter,
|
||||
fake_impl=_sp_local_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="sp_shard",
|
||||
op_func=_sp_shard_op,
|
||||
fake_impl=_sp_local_fake,
|
||||
)
|
||||
|
||||
|
||||
def sp_reduce_scatter(x: torch.Tensor) -> torch.Tensor:
|
||||
if torch.compiler.is_compiling():
|
||||
return torch.ops.vllm.sp_reduce_scatter(x)
|
||||
return common_sp.sp_reduce_scatter(x)
|
||||
|
||||
|
||||
def sp_shard(x: torch.Tensor) -> torch.Tensor:
|
||||
if torch.compiler.is_compiling():
|
||||
return torch.ops.vllm.sp_shard(x)
|
||||
return common_sp.sp_shard(x)
|
||||
@@ -0,0 +1,883 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GPU-resident Qwen4Exp position-learning enhancement layers."""
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_etp_group, get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.model_executor.layers.mamba.abstract import MambaBase
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import is_fp8
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.short_conv_attn import (
|
||||
PleShortConvAttentionBackend,
|
||||
PleShortConvAttentionMetadata,
|
||||
)
|
||||
|
||||
from ..common.ngram_embedding import (
|
||||
Qwen4ExpPLEDeviceEmbedding,
|
||||
Qwen4ExpPLEEmbeddingMethod,
|
||||
Qwen4ExpPLENvFp4EmbeddingMethod,
|
||||
Qwen4ExpPLEPinnedHostEmbedding,
|
||||
)
|
||||
from ..common.ops.ple import ple_conv, ple_gate, ple_ngram_ids
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Qwen4ExpPLEGroupedNorm(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
eps: float,
|
||||
group_size: int | None,
|
||||
dtype: torch.dtype | None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if group_size is not None and hidden_size % group_size:
|
||||
raise ValueError(
|
||||
f"hidden_size ({hidden_size}) must be divisible by "
|
||||
f"group_size ({group_size})"
|
||||
)
|
||||
self.eps = eps
|
||||
self.group_size = group_size
|
||||
self.weight = nn.Parameter(torch.zeros(hidden_size, dtype=dtype))
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
input_dtype = hidden_states.dtype
|
||||
hidden_states = hidden_states.float()
|
||||
if self.group_size is None:
|
||||
variance = hidden_states.square().mean(dim=-1, keepdim=True)
|
||||
normalized = hidden_states * torch.rsqrt(variance + self.eps)
|
||||
else:
|
||||
grouped = hidden_states.unflatten(
|
||||
-1, (hidden_states.shape[-1] // self.group_size, self.group_size)
|
||||
)
|
||||
variance = grouped.square().mean(dim=-1, keepdim=True)
|
||||
normalized = (grouped * torch.rsqrt(variance + self.eps)).flatten(-2)
|
||||
return (normalized * (1.0 + self.weight.float())).to(input_dtype)
|
||||
|
||||
|
||||
class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
_MASK64 = (1 << 64) - 1
|
||||
_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
|
||||
_SPLITMIX_M1 = 0xBF58476D1CE4E5B9
|
||||
_SPLITMIX_M2 = 0x94D049BB133111EB
|
||||
_PLE_LAYER_PRIME = 10007
|
||||
|
||||
@classmethod
|
||||
def _splitmix64(cls, value: int) -> int:
|
||||
"""Mix an integer into a deterministic unsigned 64-bit value."""
|
||||
value = (value + cls._SPLITMIX_GAMMA) & cls._MASK64
|
||||
value = ((value ^ (value >> 30)) * cls._SPLITMIX_M1) & cls._MASK64
|
||||
value = ((value ^ (value >> 27)) * cls._SPLITMIX_M2) & cls._MASK64
|
||||
return (value ^ (value >> 31)) & cls._MASK64
|
||||
|
||||
@staticmethod
|
||||
def _is_prime_64(value: int) -> bool:
|
||||
"""Return whether a 64-bit integer is prime."""
|
||||
if value < 2:
|
||||
return False
|
||||
for prime in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37):
|
||||
if value % prime == 0:
|
||||
return value == prime
|
||||
exponent = value - 1
|
||||
shifts = 0
|
||||
while exponent % 2 == 0:
|
||||
exponent //= 2
|
||||
shifts += 1
|
||||
for base in (2, 325, 9375, 28178, 450775, 9780504, 1795265022):
|
||||
if base % value == 0:
|
||||
continue
|
||||
witness = pow(base, exponent, value)
|
||||
if witness in (1, value - 1):
|
||||
continue
|
||||
for _ in range(shifts - 1):
|
||||
witness = pow(witness, 2, value)
|
||||
if witness == value - 1:
|
||||
break
|
||||
else:
|
||||
return False
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def _nth_prime_after(cls, start: int, count: int) -> int:
|
||||
"""Return the ``count``-th prime strictly greater than ``start``."""
|
||||
prime = int(start)
|
||||
for _ in range(count):
|
||||
candidate = prime + 1
|
||||
if candidate <= 2:
|
||||
prime = 2
|
||||
continue
|
||||
if candidate % 2 == 0:
|
||||
candidate += 1
|
||||
while not cls._is_prime_64(candidate):
|
||||
candidate += 2
|
||||
prime = candidate
|
||||
return prime
|
||||
|
||||
@classmethod
|
||||
def _make_layer_multipliers(
|
||||
cls,
|
||||
*,
|
||||
ngram_size: int,
|
||||
unigram_vocab_size: int,
|
||||
seed: int,
|
||||
ple_dense_layer_id: int,
|
||||
) -> list[int]:
|
||||
"""Build deterministic hash multipliers for one PLE layer."""
|
||||
max_multiplier = ((1 << 63) - 1) // unigram_vocab_size
|
||||
half_bound = max(1, max_multiplier // 2)
|
||||
base_seed = seed + cls._PLE_LAYER_PRIME * ple_dense_layer_id
|
||||
multipliers = []
|
||||
for index in range(ngram_size):
|
||||
value = base_seed + cls._SPLITMIX_GAMMA * (index + 1)
|
||||
multipliers.append(2 * (cls._splitmix64(value) % half_bound) + 1)
|
||||
return multipliers
|
||||
|
||||
@classmethod
|
||||
def _make_vocab_layout(
|
||||
cls,
|
||||
*,
|
||||
ngram_vocab_size_base: int,
|
||||
ngram_heads: int,
|
||||
ple_dense_layer_id: int,
|
||||
) -> tuple[list[int], list[int], int]:
|
||||
"""Build per-head vocabulary sizes, offsets, and total row count."""
|
||||
sizes: list[int] = []
|
||||
offsets: list[int] = []
|
||||
offset = 0
|
||||
for local_head in range(ngram_heads):
|
||||
global_head = ple_dense_layer_id * ngram_heads + local_head
|
||||
size = cls._nth_prime_after(ngram_vocab_size_base - 1, global_head + 1)
|
||||
sizes.append(size)
|
||||
offsets.append(offset)
|
||||
offset += size
|
||||
return sizes, offsets, offset
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen4ExpTextConfig,
|
||||
embedding_dim: int,
|
||||
ple_dense_layer_id: int,
|
||||
max_total_tokens: int,
|
||||
prefix: str,
|
||||
layer_name: str,
|
||||
*,
|
||||
data_parallel_rank: int = 0,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
use_sequence_parallel: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.use_sequence_parallel = use_sequence_parallel
|
||||
self.embedding_dim = embedding_dim
|
||||
self.layer_name = layer_name
|
||||
self.ngram_size = int(config.ngram_size)
|
||||
self.heads_per_ngram = int(config.heads_per_ngram)
|
||||
self.ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram
|
||||
if self.ngram_size < 2:
|
||||
raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}")
|
||||
if self.heads_per_ngram <= 0:
|
||||
raise ValueError(f"heads_per_ngram must be > 0, got {self.heads_per_ngram}")
|
||||
if embedding_dim % self.ngram_heads:
|
||||
raise ValueError(
|
||||
"ple_embed_dim must be divisible by total ngram heads: "
|
||||
f"{embedding_dim} % {self.ngram_heads} != 0"
|
||||
)
|
||||
self.head_dim = embedding_dim // self.ngram_heads
|
||||
self.eos_token_id = int(config.eos_token_id)
|
||||
self.unigram_vocab_size = int(config.vocab_size)
|
||||
self.split_ngram_parts = int(getattr(config, "split_ngram_parts", 512))
|
||||
if self.split_ngram_parts <= 0:
|
||||
raise ValueError("split_ngram_parts must be positive")
|
||||
|
||||
multipliers = self._make_layer_multipliers(
|
||||
ngram_size=self.ngram_size,
|
||||
unigram_vocab_size=self.unigram_vocab_size,
|
||||
seed=int(getattr(config, "seed", 1234)),
|
||||
ple_dense_layer_id=ple_dense_layer_id,
|
||||
)
|
||||
self.register_buffer(
|
||||
"layer_multipliers",
|
||||
torch.tensor(multipliers, dtype=torch.long),
|
||||
persistent=True,
|
||||
)
|
||||
|
||||
sizes, offsets, total_vocab_size = self._make_vocab_layout(
|
||||
ngram_vocab_size_base=int(config.ngram_vocab_size_base),
|
||||
ngram_heads=self.ngram_heads,
|
||||
ple_dense_layer_id=ple_dense_layer_id,
|
||||
)
|
||||
self.register_buffer(
|
||||
"ngram_heads_vocab_sizes",
|
||||
torch.tensor(sizes, dtype=torch.long),
|
||||
persistent=True,
|
||||
)
|
||||
self.register_buffer(
|
||||
"ngram_heads_offsets",
|
||||
torch.tensor(offsets, dtype=torch.long),
|
||||
persistent=True,
|
||||
)
|
||||
divisor = int(config.make_ngram_vocab_size_divisible_by)
|
||||
padded_vocab_size = ((total_vocab_size + divisor - 1) // divisor) * divisor
|
||||
if params_dtype is None:
|
||||
params_dtype = torch.get_default_dtype()
|
||||
embedding_prefix = f"{prefix}.ngram_embedding"
|
||||
embedding_quant_method = Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
embedding_prefix,
|
||||
getattr(config, "ple_embedding_dtype", None),
|
||||
)
|
||||
if isinstance(embedding_quant_method, Qwen4ExpPLENvFp4EmbeddingMethod):
|
||||
raise NotImplementedError(
|
||||
"NVFP4 Qwen4Exp PLE tables are not supported on ROCm yet; use a "
|
||||
"checkpoint with a BF16 or FP8 PLE table."
|
||||
)
|
||||
engram_config = get_current_vllm_config().engram_config
|
||||
embedding_cls = (
|
||||
Qwen4ExpPLEPinnedHostEmbedding
|
||||
if engram_config is not None and engram_config.cpu_offload
|
||||
else Qwen4ExpPLEDeviceEmbedding
|
||||
)
|
||||
# A TP reduce-scatter covers all vocabulary owners only when ETP=TP.
|
||||
self.use_reduce_scatter = (
|
||||
self.use_sequence_parallel
|
||||
and get_etp_group().world_size == get_tp_group().world_size
|
||||
)
|
||||
self.ngram_embedding = embedding_cls(
|
||||
padded_vocab_size,
|
||||
self.head_dim,
|
||||
params_dtype=params_dtype,
|
||||
padding_size=divisor,
|
||||
prefix=embedding_prefix,
|
||||
embedding_method=embedding_quant_method,
|
||||
num_ngram_heads=self.ngram_heads,
|
||||
max_total_tokens=max_total_tokens,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
reduce_results=not self.use_reduce_scatter,
|
||||
)
|
||||
logger.info(
|
||||
"Initialized AMD PLE embedding %s: quantization_method=%s, "
|
||||
"weight_dtype=%s, weight_device=%s, pinned=%s",
|
||||
embedding_prefix,
|
||||
type(embedding_quant_method).__name__,
|
||||
self.ngram_embedding.weight.dtype,
|
||||
self.ngram_embedding.weight.device,
|
||||
self.ngram_embedding.weight.is_pinned(),
|
||||
)
|
||||
|
||||
def _sp_shard_embeddings(self, embeddings: torch.Tensor) -> torch.Tensor:
|
||||
"""Shard embedding token rows, optionally reducing vocabulary contributions."""
|
||||
# Each embedding element has one vocabulary owner, including FP8 bytes.
|
||||
output_dtype = embeddings.dtype
|
||||
if is_fp8(embeddings):
|
||||
embeddings = embeddings.view(torch.int8)
|
||||
if self.use_reduce_scatter:
|
||||
embeddings = sp_reduce_scatter(embeddings)
|
||||
else:
|
||||
embeddings = sp_shard(embeddings)
|
||||
return embeddings.view(output_dtype)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Return full embeddings or a token shard after combining vocabulary owners."""
|
||||
input_ids = input_ids.reshape(-1)
|
||||
ngram_ids = input_ids.new_empty(
|
||||
(input_ids.shape[0], self.ngram_heads), dtype=torch.int64
|
||||
)
|
||||
torch.ops.vllm.qwen4_exp_amd_ple_ngram_ids(
|
||||
input_ids,
|
||||
query_start_loc,
|
||||
ngram_context,
|
||||
ngram_ids,
|
||||
self.layer_name,
|
||||
)
|
||||
embedding = self.ngram_embedding
|
||||
if embedding.supports_prefetch:
|
||||
output = ngram_ids.new_empty(
|
||||
(ngram_ids.shape[0], self.embedding_dim),
|
||||
dtype=embedding.weight.dtype,
|
||||
)
|
||||
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding_pinned(
|
||||
ngram_ids,
|
||||
output,
|
||||
self.layer_name,
|
||||
)
|
||||
else:
|
||||
output = ngram_ids.new_empty(
|
||||
(ngram_ids.shape[0], self.embedding_dim),
|
||||
dtype=embedding.params_dtype,
|
||||
)
|
||||
torch.ops.vllm.qwen4_exp_amd_ple_ngram_embedding(
|
||||
ngram_ids,
|
||||
output,
|
||||
self.layer_name,
|
||||
)
|
||||
if not self.use_sequence_parallel:
|
||||
return output
|
||||
return self._sp_shard_embeddings(output)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load hash buffers and checkpoint-split embedding rows."""
|
||||
persistent_buffers = {
|
||||
"layer_multipliers": self.layer_multipliers,
|
||||
"ngram_heads_offsets": self.ngram_heads_offsets,
|
||||
"ngram_heads_vocab_sizes": self.ngram_heads_vocab_sizes,
|
||||
}
|
||||
loaded: set[str] = set()
|
||||
regular_weights: list[tuple[str, torch.Tensor]] = []
|
||||
shard_prefix = "ngram_embedding.shard_"
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
leaf_name = name.rsplit(".", 1)[-1]
|
||||
if leaf_name.startswith("hashstats_") or leaf_name == "token_lookup":
|
||||
continue
|
||||
if name in persistent_buffers:
|
||||
buffer = persistent_buffers[name]
|
||||
if buffer.shape != loaded_weight.shape:
|
||||
raise ValueError(
|
||||
f"Shape mismatch for {name}: expected "
|
||||
f"{tuple(buffer.shape)}, got {tuple(loaded_weight.shape)}"
|
||||
)
|
||||
buffer.copy_(loaded_weight.to(device=buffer.device, dtype=buffer.dtype))
|
||||
loaded.add(name)
|
||||
continue
|
||||
if name.startswith(shard_prefix) and name.endswith(".weight"):
|
||||
shard_text = name[len(shard_prefix) : -len(".weight")]
|
||||
if not shard_text.isdigit():
|
||||
regular_weights.append((name, loaded_weight))
|
||||
continue
|
||||
shard_index = int(shard_text)
|
||||
if shard_index >= self.split_ngram_parts:
|
||||
raise ValueError(
|
||||
f"PLE embedding shard index {shard_index} exceeds "
|
||||
f"split_ngram_parts={self.split_ngram_parts}"
|
||||
)
|
||||
embedding = self.ngram_embedding
|
||||
shard_size = (
|
||||
embedding.org_vocab_size + self.split_ngram_parts - 1
|
||||
) // self.split_ngram_parts
|
||||
checkpoint_start = shard_index * shard_size
|
||||
expected_rows = max(
|
||||
0,
|
||||
min(shard_size, embedding.org_vocab_size - checkpoint_start),
|
||||
)
|
||||
expected_shape = (expected_rows, embedding.embedding_dim)
|
||||
if tuple(loaded_weight.shape) != expected_shape:
|
||||
raise ValueError(
|
||||
f"Shape mismatch for PLE embedding shard {shard_index}: "
|
||||
f"expected {expected_shape}, got "
|
||||
f"{tuple(loaded_weight.shape)}"
|
||||
)
|
||||
embedding.weight.weight_loader(
|
||||
embedding.weight,
|
||||
loaded_weight,
|
||||
checkpoint_start=checkpoint_start,
|
||||
)
|
||||
loaded.add("ngram_embedding.weight")
|
||||
continue
|
||||
regular_weights.append((name, loaded_weight))
|
||||
|
||||
if regular_weights:
|
||||
loaded.update(AutoWeightsLoader(self).load_weights(regular_weights))
|
||||
return loaded
|
||||
|
||||
|
||||
class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen4ExpTextConfig,
|
||||
vllm_config: VllmConfig,
|
||||
layer_idx: int = 0,
|
||||
ple_dense_layer_id: int | None = None,
|
||||
prefix: str = "",
|
||||
use_sequence_parallel: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.use_sequence_parallel = use_sequence_parallel
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.model_config: ModelConfig = model_config
|
||||
self.cache_config: CacheConfig = cache_config
|
||||
self.layer_idx = layer_idx
|
||||
self.ple_dense_layer_id = (
|
||||
int(ple_dense_layer_id)
|
||||
if ple_dense_layer_id is not None
|
||||
else int(layer_idx)
|
||||
)
|
||||
self.prefix = prefix
|
||||
self.hidden_size = int(config.hidden_size)
|
||||
self.hc_count = config.hc_count
|
||||
self.hc_hidden_size = self.hidden_size * self.hc_count
|
||||
self.conv_kernel_size = int(config.ple_conv_kernel_size)
|
||||
self.short_conv_dilation = int(config.ngram_size)
|
||||
self.conv_state_len = (self.conv_kernel_size - 1) * self.short_conv_dilation
|
||||
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
||||
self.activation = "silu"
|
||||
self.ple_embedding: nn.Module = Qwen4ExpNGramEmbedding(
|
||||
config,
|
||||
int(config.ple_embed_dim),
|
||||
self.ple_dense_layer_id,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
f"{prefix}.ple_embedding",
|
||||
prefix,
|
||||
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
|
||||
quant_config=quant_config,
|
||||
params_dtype=model_config.dtype,
|
||||
use_sequence_parallel=self.use_sequence_parallel,
|
||||
)
|
||||
# Projection weights stay replicated even when their token rows are sharded.
|
||||
self.kv_proj = MergedColumnParallelLinear(
|
||||
int(config.ple_embed_dim),
|
||||
[self.hc_hidden_size, self.hidden_size],
|
||||
bias=False,
|
||||
params_dtype=model_config.dtype,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.kv_proj",
|
||||
disable_tp=True,
|
||||
)
|
||||
norm_args = (
|
||||
self.hc_hidden_size,
|
||||
config.rms_norm_eps,
|
||||
self.hidden_size,
|
||||
model_config.dtype,
|
||||
)
|
||||
self.norm_key = Qwen4ExpPLEGroupedNorm(*norm_args)
|
||||
self.norm_query = Qwen4ExpPLEGroupedNorm(*norm_args)
|
||||
self.norm_conv = Qwen4ExpPLEGroupedNorm(*norm_args)
|
||||
self.conv1d = nn.Conv1d(
|
||||
self.hc_hidden_size,
|
||||
self.hc_hidden_size,
|
||||
self.conv_kernel_size,
|
||||
groups=self.hc_hidden_size,
|
||||
padding=self.conv_state_len,
|
||||
dilation=self.short_conv_dilation,
|
||||
bias=False,
|
||||
dtype=model_config.dtype,
|
||||
)
|
||||
nn.init.zeros_(self.conv1d.weight)
|
||||
self.conv1d.weight._no_reinit = True
|
||||
self.kv_cache = (torch.tensor([]),)
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.SHORT_CONV
|
||||
|
||||
@property
|
||||
def is_kv_cache_tp_replicated(self) -> bool:
|
||||
return True
|
||||
|
||||
def get_attn_backend(self) -> type[PleShortConvAttentionBackend]:
|
||||
return PleShortConvAttentionBackend
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
return MambaStateDtypeCalculator.short_conv_state_dtype(
|
||||
self.model_config.dtype, self.cache_config.mamba_cache_dtype
|
||||
)
|
||||
|
||||
def get_state_shape(self) -> Sequence[tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.short_conv_state_shape(
|
||||
tp_world_size=1,
|
||||
intermediate_size=self.hc_hidden_size,
|
||||
conv_kernel=self.conv_state_len + 1,
|
||||
num_spec=self.num_spec_tokens,
|
||||
)
|
||||
|
||||
def _short_conv_dilated_dispatch(
|
||||
self,
|
||||
inputs: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor | None,
|
||||
metadata: PleShortConvAttentionMetadata,
|
||||
conv_state: torch.Tensor,
|
||||
conv_weights: torch.Tensor,
|
||||
) -> None:
|
||||
num_prefills = metadata.num_prefills
|
||||
num_decodes = metadata.num_decodes
|
||||
num_decode_tokens = metadata.num_decode_tokens
|
||||
num_prefill_tokens = metadata.num_prefill_tokens
|
||||
has_prefill = num_prefills > 0
|
||||
has_decode = num_decodes > 0
|
||||
has_spec = metadata.spec_sequence_masks is not None
|
||||
has_non_spec = has_prefill or has_decode
|
||||
inputs = inputs[: metadata.num_actual_tokens]
|
||||
residual = residual[: metadata.num_actual_tokens]
|
||||
if outer_residual is not None:
|
||||
outer_residual = outer_residual[: metadata.num_actual_tokens]
|
||||
|
||||
spec_token_indices = None
|
||||
non_spec_token_indices = None
|
||||
if has_spec and has_non_spec:
|
||||
assert metadata.spec_token_indx is not None
|
||||
assert metadata.non_spec_token_indx is not None
|
||||
spec_token_indices = metadata.spec_token_indx
|
||||
non_spec_token_indices = metadata.non_spec_token_indx
|
||||
|
||||
if has_spec:
|
||||
assert metadata.spec_state_indices_tensor is not None
|
||||
query_start_loc = metadata.spec_query_start_loc
|
||||
num_accepted_tokens = metadata.num_accepted_tokens
|
||||
assert query_start_loc is not None
|
||||
assert num_accepted_tokens is not None
|
||||
spec_state_indices = metadata.spec_state_indices_tensor[
|
||||
: metadata.num_spec_decodes
|
||||
]
|
||||
# Mixed batches stay in their original row order; the kernels map
|
||||
# logical spec/non-spec rows instead of materializing both groups.
|
||||
ple_conv(
|
||||
inputs=inputs,
|
||||
residual=residual,
|
||||
conv_state=conv_state,
|
||||
conv_weights=conv_weights,
|
||||
state_indices=spec_state_indices,
|
||||
outer_residual=outer_residual,
|
||||
mode="spec",
|
||||
dilation=self.short_conv_dilation,
|
||||
query_start_loc=query_start_loc,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
spec_query_len=metadata.spec_query_len,
|
||||
token_indices=spec_token_indices,
|
||||
)
|
||||
|
||||
if not has_non_spec:
|
||||
return
|
||||
|
||||
state_indices = metadata.state_indices_tensor
|
||||
assert state_indices is not None
|
||||
if has_prefill:
|
||||
state_indices_d, state_indices_p = torch.split(
|
||||
state_indices, [num_decodes, num_prefills], dim=0
|
||||
)
|
||||
if non_spec_token_indices is None:
|
||||
inputs_d, inputs_p = torch.split(
|
||||
inputs, [num_decode_tokens, num_prefill_tokens], dim=0
|
||||
)
|
||||
residual_d, residual_p = torch.split(
|
||||
residual, [num_decode_tokens, num_prefill_tokens], dim=0
|
||||
)
|
||||
outer_residual_d: torch.Tensor | None = None
|
||||
outer_residual_p: torch.Tensor | None = None
|
||||
if outer_residual is not None:
|
||||
outer_residual_d, outer_residual_p = torch.split(
|
||||
outer_residual,
|
||||
[num_decode_tokens, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
token_indices_d = None
|
||||
token_indices_p = None
|
||||
else:
|
||||
inputs_d = inputs_p = inputs
|
||||
residual_d = residual_p = residual
|
||||
outer_residual_d = outer_residual_p = outer_residual
|
||||
token_indices_d, token_indices_p = torch.split(
|
||||
non_spec_token_indices,
|
||||
[num_decode_tokens, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
|
||||
if has_decode:
|
||||
ple_conv(
|
||||
inputs=inputs_d,
|
||||
residual=residual_d,
|
||||
conv_state=conv_state,
|
||||
conv_weights=conv_weights,
|
||||
state_indices=state_indices_d,
|
||||
outer_residual=outer_residual_d,
|
||||
mode="decode",
|
||||
dilation=self.short_conv_dilation,
|
||||
has_initial_states=metadata.has_initial_states_d,
|
||||
token_indices=token_indices_d,
|
||||
)
|
||||
|
||||
query_start_loc = metadata.query_start_loc_p
|
||||
if query_start_loc is None:
|
||||
raise ValueError("query_start_loc is required for prefill short-conv")
|
||||
has_initial_states = metadata.has_initial_states_p
|
||||
if has_initial_states is None:
|
||||
raise ValueError(
|
||||
"has_initial_states_p is required for prefill short-conv"
|
||||
)
|
||||
ple_conv(
|
||||
inputs=inputs_p,
|
||||
residual=residual_p,
|
||||
conv_state=conv_state,
|
||||
conv_weights=conv_weights,
|
||||
state_indices=state_indices_p,
|
||||
outer_residual=outer_residual_p,
|
||||
mode="prefill",
|
||||
dilation=self.short_conv_dilation,
|
||||
query_start_loc=query_start_loc,
|
||||
has_initial_states=has_initial_states,
|
||||
token_indices=token_indices_p,
|
||||
)
|
||||
else:
|
||||
num_decode_rows = (
|
||||
non_spec_token_indices.numel()
|
||||
if non_spec_token_indices is not None
|
||||
else inputs.size(0)
|
||||
)
|
||||
ple_conv(
|
||||
inputs=inputs,
|
||||
residual=residual,
|
||||
conv_state=conv_state,
|
||||
conv_weights=conv_weights,
|
||||
state_indices=state_indices[:num_decode_rows],
|
||||
outer_residual=outer_residual,
|
||||
mode="decode",
|
||||
dilation=self.short_conv_dilation,
|
||||
has_initial_states=metadata.has_initial_states_d,
|
||||
token_indices=non_spec_token_indices,
|
||||
)
|
||||
|
||||
def _short_conv(
|
||||
self,
|
||||
inputs: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor | None,
|
||||
) -> None:
|
||||
"""Update convolution state, optionally fusing both residual additions."""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
# Profiling omits all metadata or this Mamba entry. Short convolution
|
||||
# is a no-op there, but preserve the outer residual addition.
|
||||
if attn_metadata is None:
|
||||
if outer_residual is not None:
|
||||
residual.add_(outer_residual)
|
||||
return
|
||||
|
||||
if not isinstance(attn_metadata, dict):
|
||||
raise RuntimeError(
|
||||
"PLE short-conv expects per-layer attention metadata dict "
|
||||
f"during inference, got {type(attn_metadata).__name__}."
|
||||
)
|
||||
|
||||
layer_attn_metadata = attn_metadata.get(self.prefix)
|
||||
if layer_attn_metadata is None:
|
||||
if outer_residual is not None:
|
||||
residual.add_(outer_residual)
|
||||
return
|
||||
if not isinstance(layer_attn_metadata, PleShortConvAttentionMetadata):
|
||||
raise TypeError(
|
||||
"Expected PleShortConvAttentionMetadata for layer "
|
||||
f"'{self.prefix}', got "
|
||||
f"{type(layer_attn_metadata).__name__}."
|
||||
)
|
||||
|
||||
conv_state = self.kv_cache[0]
|
||||
# Canonicalize both backend cache layouts to [slot, channel, window].
|
||||
if not is_conv_state_dim_first():
|
||||
conv_state = conv_state.transpose(-1, -2)
|
||||
conv_weights = self.conv1d.weight.squeeze(1)
|
||||
|
||||
state_capacity = self.conv_state_len + self.num_spec_tokens
|
||||
if state_capacity > 0:
|
||||
if conv_state.size(-1) < state_capacity:
|
||||
raise RuntimeError(
|
||||
"PLE short-conv cache is smaller than expected for "
|
||||
f"dilated convolution: got {conv_state.size(-1)}, "
|
||||
f"expect at least {state_capacity}."
|
||||
)
|
||||
conv_state = conv_state[..., -state_capacity:]
|
||||
self._short_conv_dilated_dispatch(
|
||||
inputs=inputs,
|
||||
residual=residual,
|
||||
outer_residual=outer_residual,
|
||||
metadata=layer_attn_metadata,
|
||||
conv_state=conv_state,
|
||||
conv_weights=conv_weights.to(dtype=inputs.dtype),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
input_ids = input_ids.reshape(-1)
|
||||
if (
|
||||
not self.use_sequence_parallel
|
||||
and input_ids.shape[0] != hidden_states.shape[0]
|
||||
):
|
||||
raise ValueError(
|
||||
"PLE expects input_ids and hidden_states to have the same "
|
||||
f"token length, got {input_ids.shape[0]} and "
|
||||
f"{hidden_states.shape[0]}"
|
||||
)
|
||||
embeddings = self.ple_embedding(input_ids, query_start_loc, ngram_context)
|
||||
embeddings = self.ple_embedding.ngram_embedding.dequantize(
|
||||
embeddings, hidden_states.dtype
|
||||
)
|
||||
kv, _ = self.kv_proj(embeddings)
|
||||
key, value = kv.split(self.kv_proj.output_sizes, dim=-1)
|
||||
gated_output, conv_input = ple_gate(
|
||||
key,
|
||||
value,
|
||||
hidden_states,
|
||||
self.norm_key.weight,
|
||||
self.norm_query.weight,
|
||||
self.norm_conv.weight,
|
||||
self.norm_key.eps,
|
||||
)
|
||||
# The short-conv op is a piecewise-graph splitting op: state routing
|
||||
# reads runtime request metadata. It accumulates the convolution and
|
||||
# the outer residual into gated_output.
|
||||
if self.use_sequence_parallel:
|
||||
# Only convolution needs full tokens. Add residuals on local shards.
|
||||
conv_input = sp_all_gather(conv_input)[: input_ids.shape[0]]
|
||||
# Metadata may omit padding tokens or all tokens during profiling.
|
||||
conv_output = torch.zeros_like(conv_input)
|
||||
torch.ops.vllm.qwen4_exp_ple_short_conv(
|
||||
conv_input, conv_output, None, self.prefix
|
||||
)
|
||||
gated_output.add_(sp_shard(conv_output))
|
||||
gated_output.add_(hidden_states)
|
||||
else:
|
||||
torch.ops.vllm.qwen4_exp_ple_short_conv(
|
||||
conv_input,
|
||||
gated_output,
|
||||
hidden_states,
|
||||
self.prefix,
|
||||
)
|
||||
return gated_output
|
||||
|
||||
|
||||
def qwen4_exp_amd_ple_ngram_ids(
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Hash the current request layout into n-gram embedding indices.
|
||||
|
||||
The launch sizes its request binary search from the request count, which
|
||||
is symbolic under Dynamo, so it runs outside Inductor's FX graph.
|
||||
"""
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
if not isinstance(layer, Qwen4ExpPLELayer):
|
||||
raise TypeError(f"{layer_name} is not a Qwen4Exp PLE owner")
|
||||
embedding = layer.ple_embedding
|
||||
ple_ngram_ids(
|
||||
input_ids=input_ids,
|
||||
query_start_loc=query_start_loc,
|
||||
ngram_context=ngram_context,
|
||||
layer_multipliers=embedding.layer_multipliers,
|
||||
ngram_heads_vocab_sizes=embedding.ngram_heads_vocab_sizes,
|
||||
ngram_heads_offsets=embedding.ngram_heads_offsets,
|
||||
eos_token_id=embedding.eos_token_id,
|
||||
heads_per_ngram=embedding.heads_per_ngram,
|
||||
output=output,
|
||||
)
|
||||
|
||||
|
||||
def qwen4_exp_amd_ple_ngram_embedding(
|
||||
ngram_ids: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Run the large PLE embedding lookup outside Inductor's FX graph.
|
||||
|
||||
Keeping the embedding weight in ``static_forward_context`` prevents AOT
|
||||
compile-time autotuning from materializing a synthetic copy of the weight.
|
||||
"""
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
if not isinstance(layer, Qwen4ExpPLELayer):
|
||||
raise TypeError(f"{layer_name} is not a Qwen4Exp PLE owner")
|
||||
result = layer.ple_embedding.ngram_embedding(ngram_ids).flatten(-2)
|
||||
output.copy_(result)
|
||||
|
||||
|
||||
def qwen4_exp_amd_ple_ngram_embedding_pinned(
|
||||
ngram_ids: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Run the pinned PLE UVA lookup outside Inductor's FX graph.
|
||||
|
||||
Same rationale as the device-path escape: keeping the large embedding weight
|
||||
out of the graph prevents AOT compile-time autotuning from materializing a
|
||||
synthetic copy of the weight.
|
||||
"""
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
if not isinstance(layer, Qwen4ExpPLELayer):
|
||||
raise TypeError(f"{layer_name} is not a Qwen4Exp PLE owner")
|
||||
result = layer.ple_embedding.ngram_embedding.sync_lookup(ngram_ids).flatten(-2)
|
||||
output.copy_(result)
|
||||
|
||||
|
||||
def qwen4_exp_ple_short_conv(
|
||||
inputs: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
outer_residual: torch.Tensor | None,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
layer._short_conv(inputs, residual, outer_residual)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_amd_ple_ngram_ids",
|
||||
op_func=qwen4_exp_amd_ple_ngram_ids,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_amd_ple_ngram_embedding",
|
||||
op_func=qwen4_exp_amd_ple_ngram_embedding,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_amd_ple_ngram_embedding_pinned",
|
||||
op_func=qwen4_exp_amd_ple_ngram_embedding_pinned,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_ple_short_conv",
|
||||
op_func=qwen4_exp_ple_short_conv,
|
||||
mutates_args=["residual"],
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Qwen4ExpNGramEmbedding",
|
||||
"Qwen4ExpPLEGroupedNorm",
|
||||
"Qwen4ExpPLELayer",
|
||||
]
|
||||
@@ -0,0 +1,544 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""AMD ROCm QSA owner with Triton kernels."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import ClassVar, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.attention.attention import (
|
||||
set_default_quant_scales,
|
||||
)
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
||||
from vllm.model_executor.models.qwen3_vl import mrope_positions_factor
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
canonicalize_singleton_dim_strides,
|
||||
direct_register_custom_op,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionCGSupport,
|
||||
AttentionType,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.fa_utils import is_flash_attn_varlen_func_available
|
||||
from vllm.v1.attention.backends.flash_attn import (
|
||||
FlashAttentionBackend,
|
||||
FlashAttentionImpl,
|
||||
FlashAttentionMetadata,
|
||||
FlashAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheSpec,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
from ..common.qsa_cache import QSAForwardMetadata
|
||||
from . import model
|
||||
from .indexer_qsa import QSAIndexer
|
||||
|
||||
|
||||
class Qwen4ExpQSAMetadataBuilder(FlashAttentionMetadataBuilder):
|
||||
"""Flash metadata supporting uniform decode and target-verify graphs."""
|
||||
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
|
||||
class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
|
||||
"""FullAttentionSpec backend used by the merged QSA owner."""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "QWEN4_EXP_QSA_TRITON"
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
||||
# QSA consumes manager pages directly and does not use FA4 paged attention.
|
||||
return [MultipleOf(16)]
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type[Qwen4ExpQSAFlashAttentionImpl]:
|
||||
return Qwen4ExpQSAFlashAttentionImpl
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type[Qwen4ExpQSAMetadataBuilder]:
|
||||
return Qwen4ExpQSAMetadataBuilder
|
||||
|
||||
@classmethod
|
||||
def is_sparse(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_kv_connector(cls) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
||||
"""Run paged sparse GQA with the QSA Triton kernel."""
|
||||
|
||||
supports_dcp: bool = False
|
||||
supports_pcp: bool = False
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
if not is_flash_attn_varlen_func_available():
|
||||
raise NotImplementedError("Qwen4Exp QSA requires FlashAttention")
|
||||
if self.dcp_world_size != 1:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA does not support decode context parallelism"
|
||||
)
|
||||
if self.kv_cache_dtype not in ("auto", "bfloat16"):
|
||||
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
|
||||
self.supports_quant_query_input = False
|
||||
|
||||
def forward_qsa(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None,
|
||||
value: torch.Tensor | None,
|
||||
kv_cache: torch.Tensor,
|
||||
attn_metadata: FlashAttentionMetadata,
|
||||
output: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
output_block_scale: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
del key, value
|
||||
if output_scale is not None or output_block_scale is not None:
|
||||
raise NotImplementedError("QSA does not support fused output quantization")
|
||||
if self.alibi_slopes is not None or self.sinks is not None:
|
||||
raise NotImplementedError("QSA does not support ALiBi or attention sinks")
|
||||
if self.sliding_window != (-1, -1):
|
||||
raise NotImplementedError("QSA does not support sliding-window attention")
|
||||
|
||||
num_tokens = attn_metadata.num_actual_tokens
|
||||
output.zero_()
|
||||
if num_tokens == 0:
|
||||
return output
|
||||
|
||||
topk_buffer = getattr(layer, "topk_indices_buffer", None)
|
||||
if topk_buffer is None:
|
||||
raise RuntimeError("QSA owner did not provide its top-k buffer")
|
||||
logical_indices = topk_buffer[:num_tokens]
|
||||
token_to_req = token_to_req[:num_tokens]
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
key_cache = canonicalize_singleton_dim_strides(key_cache)
|
||||
value_cache = canonicalize_singleton_dim_strides(value_cache)
|
||||
if key_cache.dtype != torch.bfloat16 or query.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA requires BF16 Q/K/V")
|
||||
|
||||
from .ops.qsa import qsa_sparse_paged_attention
|
||||
|
||||
qsa_sparse_paged_attention(
|
||||
query[:num_tokens],
|
||||
key_cache,
|
||||
value_cache,
|
||||
logical_indices,
|
||||
attn_metadata.block_table,
|
||||
token_to_req,
|
||||
output[:num_tokens],
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
"""Merged Qwen full-attention owner with a QSA index side branch."""
|
||||
|
||||
supports_dcp = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
config: Qwen4ExpTextConfig,
|
||||
layer_id: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
nn.Module.__init__(self)
|
||||
cache_config = vllm_config.cache_config
|
||||
model_config = vllm_config.model_config
|
||||
if cache_config is None:
|
||||
raise ValueError("Qwen4Exp QSA requires a paged KV cache")
|
||||
if model_config.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
|
||||
if cache_config.cache_dtype not in ("auto", "bfloat16"):
|
||||
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
|
||||
if getattr(quant_config, "kv_cache_scheme", None) is not None:
|
||||
raise NotImplementedError("Qwen4Exp QSA does not support KV quantization")
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if (
|
||||
parallel_config.prefill_context_parallel_size > 1
|
||||
or parallel_config.decode_context_parallel_size > 1
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA does not support context parallelism"
|
||||
)
|
||||
if not getattr(config, "is_causal", True):
|
||||
raise NotImplementedError("Qwen4Exp QSA requires causal decoder attention")
|
||||
|
||||
self.config = config
|
||||
self.hidden_size = int(config.hidden_size)
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.total_num_heads = int(config.num_attention_heads)
|
||||
if self.total_num_heads % tp_size:
|
||||
raise ValueError("QSA attention heads must be divisible by TP size")
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.total_num_kv_heads = int(config.num_key_value_heads)
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
if self.total_num_kv_heads % tp_size:
|
||||
raise ValueError("QSA KV heads must be divisible by TP size")
|
||||
elif tp_size % self.total_num_kv_heads:
|
||||
raise ValueError("TP size must be divisible by replicated QSA KV heads")
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
self.head_dim = int(config.head_dim or self.hidden_size // self.total_num_heads)
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
self.scaling = self.head_dim**-0.5
|
||||
self.dual_chunk_attention_config = getattr(
|
||||
config, "dual_chunk_attention_config", None
|
||||
)
|
||||
if self.dual_chunk_attention_config is not None:
|
||||
raise NotImplementedError("Qwen4Exp QSA does not support dual-chunk RoPE")
|
||||
# Qwen4Exp full-attention checkpoints always pack a sigmoid output
|
||||
# gate next to Q, even when an inherited config default says otherwise.
|
||||
self.attn_output_gate = True
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
self.total_num_heads * (1 + self.attn_output_gate),
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=model.without_modelopt_fp4(quant_config),
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * self.head_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
reduce_results=reduce_results,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
self.rotary_emb = get_rope(
|
||||
head_size=self.head_dim,
|
||||
max_position=config.max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
mrope_positions_factor=mrope_positions_factor(vllm_config),
|
||||
)
|
||||
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
mm_config = model_config.multimodal_config
|
||||
text_only = mm_config is None or mm_config.language_model_only
|
||||
self.use_fused_qk_norm_rope_gate = (
|
||||
self.attn_output_gate
|
||||
and getattr(self.rotary_emb, "is_neox_style", False)
|
||||
and current_platform.is_cuda()
|
||||
and text_only
|
||||
)
|
||||
|
||||
self.layer_name = f"{prefix}.attn"
|
||||
self.attn_type = AttentionType.DECODER
|
||||
self.kv_cache_dtype = cache_config.cache_dtype
|
||||
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
|
||||
self.kv_cache_dtype, model_config
|
||||
)
|
||||
if self.kv_cache_torch_dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA requires BF16 cache storage")
|
||||
self.kv_sharing_target_layer_name = None
|
||||
self.kv_cache = torch.tensor([])
|
||||
set_default_quant_scales(self, register_buffer=True)
|
||||
|
||||
self.attn_backend = Qwen4ExpQSAFlashAttentionBackend
|
||||
self.impl = Qwen4ExpQSAFlashAttentionImpl(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
self.num_kv_heads,
|
||||
None,
|
||||
None,
|
||||
self.kv_cache_dtype,
|
||||
None,
|
||||
AttentionType.DECODER,
|
||||
None,
|
||||
)
|
||||
self.indexer = QSAIndexer(
|
||||
vllm_config=vllm_config,
|
||||
config=config,
|
||||
layer_id=layer_id,
|
||||
rotary_emb=self.rotary_emb,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.indexer",
|
||||
)
|
||||
self.use_fused_qsa_prepare = (
|
||||
self.attn_output_gate
|
||||
and self.indexer.use_fused_pre_indexer
|
||||
and self.head_dim & (self.head_dim - 1) == 0
|
||||
)
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
self.register_buffer(
|
||||
"topk_indices_buffer",
|
||||
torch.empty(
|
||||
max_tokens,
|
||||
self.indexer.output_width,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
persistent=False,
|
||||
)
|
||||
|
||||
static_context = vllm_config.compilation_config.static_forward_context
|
||||
if self.layer_name in static_context:
|
||||
raise ValueError(f"Duplicate layer name: {self.layer_name}")
|
||||
static_context[self.layer_name] = self
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return self.attn_backend
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
return FullAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_dim,
|
||||
head_size_v=self.head_dim,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
)
|
||||
|
||||
def _run_qsa(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor | None,
|
||||
key: torch.Tensor | None,
|
||||
value: torch.Tensor | None,
|
||||
output: torch.Tensor,
|
||||
qkv: torch.Tensor | None = None,
|
||||
gate: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
# query/key/value are None when the fused prepare runs inside the
|
||||
# indexer launch, which then writes the gate into ``gate``.
|
||||
metadata = get_forward_context().attn_metadata
|
||||
if isinstance(metadata, list):
|
||||
metadata = metadata[0]
|
||||
if not isinstance(metadata, dict):
|
||||
output.zero_()
|
||||
if gate is not None:
|
||||
gate.zero_()
|
||||
return
|
||||
main_metadata = cast(FlashAttentionMetadata, metadata[self.layer_name])
|
||||
if self.kv_cache.numel() == 0:
|
||||
raise RuntimeError("QSA main K/V cache is not bound")
|
||||
|
||||
num_tokens = main_metadata.num_actual_tokens
|
||||
side_metadata = cast(
|
||||
QSAForwardMetadata,
|
||||
metadata[self.indexer.raw_key_cache.prefix],
|
||||
)
|
||||
if side_metadata.num_actual_tokens != num_tokens:
|
||||
raise RuntimeError("QSA main and side metadata token counts disagree")
|
||||
selected, main_outputs = self.indexer(
|
||||
hidden_states,
|
||||
positions,
|
||||
self.topk_indices_buffer[:num_tokens],
|
||||
attn=self,
|
||||
qkv=qkv,
|
||||
slot_mapping=main_metadata.slot_mapping,
|
||||
gate_out=gate,
|
||||
)
|
||||
if selected.shape != (
|
||||
num_tokens,
|
||||
self.indexer.output_width,
|
||||
):
|
||||
raise RuntimeError("QSA indexer returned an invalid selection shape")
|
||||
impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl)
|
||||
if main_outputs is None:
|
||||
assert key is not None and value is not None
|
||||
impl.do_kv_cache_update(
|
||||
self,
|
||||
key,
|
||||
value,
|
||||
self.kv_cache,
|
||||
main_metadata.slot_mapping,
|
||||
)
|
||||
else:
|
||||
query, _ = main_outputs
|
||||
# The gate is applied to every row outside this op; padded rows
|
||||
# must stay finite so they keep the zero output forward_qsa gives.
|
||||
assert gate is not None
|
||||
gate[num_tokens:].zero_()
|
||||
assert query is not None
|
||||
impl.forward_qsa(
|
||||
self,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
self.kv_cache,
|
||||
main_metadata,
|
||||
output,
|
||||
token_to_req=side_metadata.token_to_req,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
num_tokens = hidden_states.shape[0]
|
||||
encoded_layer_name = _encode_layer_name(self.layer_name)
|
||||
if self.use_fused_qsa_prepare:
|
||||
# Norm/RoPE/gate and the K/V cache write happen inside the op.
|
||||
attn_output = qkv.new_empty(num_tokens, self.num_heads, self.head_dim)
|
||||
gate = torch.empty_like(attn_output)
|
||||
if current_platform.opaque_attention_op():
|
||||
torch.ops.vllm.qwen4_exp_qsa_prepare_with_output(
|
||||
hidden_states,
|
||||
positions,
|
||||
qkv,
|
||||
attn_output,
|
||||
gate,
|
||||
encoded_layer_name,
|
||||
)
|
||||
else:
|
||||
qwen4_exp_qsa_prepare_with_output(
|
||||
hidden_states,
|
||||
positions,
|
||||
qkv,
|
||||
attn_output,
|
||||
gate,
|
||||
encoded_layer_name,
|
||||
)
|
||||
flat_output = attn_output.view(num_tokens, -1)
|
||||
flat_output = flat_output * torch.sigmoid(gate.view(num_tokens, -1))
|
||||
output, _ = self.o_proj(flat_output)
|
||||
return output
|
||||
|
||||
q, k, v, gate = self._project_qkv_gate(qkv, positions)
|
||||
query = q.view(num_tokens, self.num_heads, self.head_dim)
|
||||
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||
attn_output = torch.empty_like(query)
|
||||
if current_platform.opaque_attention_op():
|
||||
torch.ops.vllm.qwen4_exp_qsa_with_output(
|
||||
hidden_states,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_output,
|
||||
encoded_layer_name,
|
||||
)
|
||||
else:
|
||||
qwen4_exp_qsa_with_output(
|
||||
hidden_states,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_output,
|
||||
encoded_layer_name,
|
||||
)
|
||||
flat_output = attn_output.view(num_tokens, -1)
|
||||
if gate is not None:
|
||||
flat_output = flat_output * torch.sigmoid(gate)
|
||||
output, _ = self.o_proj(flat_output)
|
||||
return output
|
||||
|
||||
|
||||
def qwen4_exp_qsa_with_output(
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
"""Run the complete QSA state/update/attend transaction."""
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
if not isinstance(layer, Qwen4ExpQSAAttention):
|
||||
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
|
||||
layer._run_qsa(
|
||||
hidden_states,
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
output,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_qsa_with_output",
|
||||
op_func=qwen4_exp_qsa_with_output,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
|
||||
|
||||
def qwen4_exp_qsa_prepare_with_output(
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
qkv: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
gate: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
"""Run the QSA transaction with the main QK-norm/RoPE/gate fused in."""
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
if not isinstance(layer, Qwen4ExpQSAAttention):
|
||||
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
|
||||
layer._run_qsa(
|
||||
hidden_states,
|
||||
positions,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
output,
|
||||
qkv=qkv,
|
||||
gate=gate,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_qsa_prepare_with_output",
|
||||
op_func=qwen4_exp_qsa_prepare_with_output,
|
||||
mutates_args=["output", "gate"],
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"QSAIndexer",
|
||||
"Qwen4ExpQSAAttention",
|
||||
"Qwen4ExpQSAFlashAttentionBackend",
|
||||
"Qwen4ExpQSAFlashAttentionImpl",
|
||||
"qwen4_exp_qsa_prepare_with_output",
|
||||
"qwen4_exp_qsa_with_output",
|
||||
]
|
||||
Reference in New Issue
Block a user