[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,68 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import cutlass
import cutlass.cute as cute
from vllm.vllm_flash_attn.cute import utils
@cute.jit
def dense_mask_mod(
batch: cute.TensorSSA,
head: cute.TensorSSA,
q_idx: cute.TensorSSA,
kv_idx: cute.TensorSSA,
seqlen_info,
aux_tensors: list,
) -> cute.TensorSSA:
dense_mask = aux_tensors[0]
batch_idx = utils.ssa_to_scalar(batch)
q_idx = utils.ssa_to_scalar(q_idx)
kv_idx = utils.ssa_to_scalar(kv_idx)
batch_stride, query_stride, _ = dense_mask.stride
aligned_mask = cute.make_tensor(
dense_mask.iterator,
cute.make_layout(
dense_mask.shape,
stride=(
cute.assume(batch_stride, divby=4),
cute.assume(query_stride, divby=4),
1,
),
),
)
mask_row = aligned_mask[batch_idx, q_idx, None]
mask_chunks = cute.flat_divide(mask_row, (4,))
mask_chunk = mask_chunks[None, (kv_idx >> 5) >> 2]
loaded = cute.make_rmem_tensor_like(mask_chunk)
cute.autovec_copy(mask_chunk, loaded)
return cute.recast_tensor(loaded, cutlass.Uint32).load()
dense_mask_mod.__vec_size__ = 128
@cute.jit
def offset_dense_mask_mod(
batch: cute.TensorSSA,
head: cute.TensorSSA,
q_idx: cute.TensorSSA,
kv_idx: cute.TensorSSA,
seqlen_info,
aux_tensors: list,
) -> cute.TensorSSA:
dense_mask = aux_tensors[0]
batch_idx = utils.ssa_to_scalar(batch)
q_idx = utils.ssa_to_scalar(q_idx)
key_start = dense_mask[batch_idx, 0, dense_mask.shape[2] - 1]
kv_idx = utils.ssa_to_scalar(kv_idx) + key_start
word_idx = kv_idx >> 5
bit_idx = cutlass.Uint32(kv_idx & 31)
word = dense_mask[batch_idx, q_idx, word_idx]
result = cute.make_rmem_tensor(1, dtype=cutlass.Uint32)
result[0] = utils.shr_u32(cutlass.Uint32(word), bit_idx)
return result.load()
offset_dense_mask_mod.__vec_size__ = 32
@@ -119,11 +119,6 @@ if HAS_TRITON:
TritonExperts,
TritonWNA16Experts,
)
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExperts,
XPUExpertsFp8,
XPUExpertsMxFp4,
)
from vllm.model_executor.layers.fused_moe.fused_moe import (
fused_experts,
get_config_file_name,
File diff suppressed because it is too large Load Diff
@@ -1,353 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
kInt4Static,
kInt4Static32,
kMxfp4Dynamic,
kMxfp4Static,
kMxfp8Dynamic,
kMxfp8Static,
)
from vllm.platforms import current_platform
if current_platform.is_xpu():
from vllm_xpu_kernels.fused_moe_interface import XpuFusedMoe
def prepare_fp8_moe_layer_for_xpu(
w13: torch.Tensor,
w13_scale: torch.Tensor,
w2: torch.Tensor,
w2_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
if w13_scale is not None and w13_scale.ndim == 3:
w13_scale = w13_scale.transpose(-1, -2).contiguous()
if w2_scale is not None and w2_scale.ndim == 3:
w2_scale = w2_scale.transpose(-1, -2).contiguous()
return (
w13.transpose(-1, -2).contiguous(),
w13_scale,
w2.transpose(-1, -2).contiguous(),
w2_scale,
)
class XPUExperts(mk.FusedMoEExpertsModular):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
self.fused_moe_impl: XpuFusedMoe | None = None
is_xe2_or_xe3 = torch.ops._xpu_C.is_xe2_arch() or torch.ops._xpu_C.is_xe3_arch()
if not is_xe2_or_xe3:
raise NotImplementedError(
"XPUExperts is only supported on Intel Xe2/Xe3 GPUs"
)
self._expects_unquantized_inputs = is_xe2_or_xe3
@property
def expects_unquantized_inputs(self) -> bool:
return self._expects_unquantized_inputs
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_xpu()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return True
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.RELU2_NO_MUL,
]
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
return True
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(None, None),
(kFp8StaticTensorSym, None),
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
workspace1 = (0,)
workspace2 = (0,)
output = (M, K)
return (workspace1, workspace2, output)
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
if self.fused_moe_impl is None:
topk = topk_ids.size(-1)
if (
self.quant_config is not None
and self.quant_config.weight_quant_dtype == "mxfp4"
):
w1 = w1.view(torch.float4_e2m1fn_x2)
w2 = w2.view(torch.float4_e2m1fn_x2)
self.fused_moe_impl = XpuFusedMoe(
w13=w1,
w13_scales=self.w1_scale,
w13_bias=self.w1_bias,
w2=w2,
w2_scales=self.w2_scale,
w2_bias=self.w2_bias,
n_experts_per_token=topk,
activation=activation.value,
num_experts=self.moe_config.num_local_experts,
ep_rank=self.moe_config.ep_rank,
ep_size=self.moe_config.ep_size,
expert_map=expert_map,
gemm1_clamp_limit=self.gemm1_clamp_limit,
)
assert self.fused_moe_impl is not None
if self.moe_config.ep_size > 1:
# Avoid topk_ids = -1, remap them to an
# expert not owned by this rank so they are skipped.
sentinel = global_num_experts - 1 if self.moe_config.ep_rank == 0 else 0
topk_ids = torch.where(topk_ids < 0, sentinel, topk_ids)
self.fused_moe_impl.apply(
output=output,
hidden_states=hidden_states,
topk_weights=topk_weights,
topk_ids=topk_ids,
a1q_scale=a1q_scale,
)
class XPUExpertsFp8(XPUExperts):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kFp8StaticTensorSym, None),
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
class XPUExpertsMxFp8(XPUExpertsFp8):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
assert quant_config.quant_dtype == "mxfp8"
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kMxfp8Static, None),
(kMxfp8Static, kMxfp8Dynamic),
]
return (weight_key, activation_key) in SUPPORTED_W_A
class XPUExpertsBlockFp8(XPUExperts):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
class XPUExpertsWNA16(XPUExperts):
"""W4A16 INT4-symmetric MoE backed by `xpu_fused_moe(is_int4=True)`.
Weight layout when `is_int4=True` (per `xpu_fused_moe` docstring):
w13: [num_experts, 2*inter_size, hidden_size] contiguous int4-packed
w13_scales: [num_experts, 2*inter_size, hidden_size // group_size]
w2: [num_experts, hidden_size, inter_size] contiguous int4-packed
w2_scales: [num_experts, hidden_size, inter_size // group_size]
Pairs with `INCXPULinearMethod` for the linear layers; together they
cover full-attn + MoE on Intel XPU end-to-end without IPEX.
"""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) in (
(kInt4Static, None),
(kInt4Static32, None),
)
class XPUExpertsMxFp4(XPUExperts):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# K = a1q.size(-1). When activations are pre-quantized packed mxfp4,
# K is the packed hidden_size (= logical / 2); the kernel output is at
# logical hidden_size (2 * K). When unquantized (bf16), K is already
# the logical size.
logical_K = K if self.expects_unquantized_inputs else 2 * K
return (0,), (0,), (M, logical_K)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kMxfp4Static, None),
(kMxfp4Static, kMxfp4Dynamic),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@@ -241,21 +241,8 @@ def backend_to_kernel_cls(
return [CutlassExpertsMxfp4]
elif backend == Mxfp4MoeBackend.TRITON:
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
OAITritonExperts,
OAITritonMxfp4ExpertsMonolithic,
)
# NOTE: prefer Monolithic > Modular, so return Monolithic first.
return [OAITritonMxfp4ExpertsMonolithic, OAITritonExperts]
elif backend == Mxfp4MoeBackend.TRITON_UNFUSED:
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
UnfusedOAITritonExperts,
)
return [UnfusedOAITritonExperts]
elif backend in (Mxfp4MoeBackend.TRITON, Mxfp4MoeBackend.TRITON_UNFUSED):
raise ImportError("gpt_oss triton_kernels experts were removed in this build.")
elif backend == Mxfp4MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
@@ -1,213 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Producer side of the QuantizedActivation contract for activation layers.
Given an activation module and the downstream linear it feeds, fuse the
activation with that linear's input quantization into a single kernel when the
linear advertises a consumable input quantization key (see quant_activation.py).
Falls back to the plain activation when nothing matches, so a model forward can
always call maybe_fused_act_quant unconditionally.
This is the manual-fusion counterpart to ActivationQuantFusionPass: when fusion
fires here, the activation and quantization are already consumed, so a compiler
pass cannot fuse the same boundary again.
"""
from collections.abc import Callable
import torch
from vllm.model_executor.layers.activation import ReLUSquaredActivation, SiluAndMul
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
get_input_quant_key,
)
from vllm.model_executor.layers.fusion.relu2_fp8_quant import (
relu_squared_static_fp8_quant,
)
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
FP8_DTYPE = current_platform.fp8_dtype()
FP4_DTYPE = torch.uint8
def _silu_and_mul_fp8_static(
x: torch.Tensor, linear: LinearBase
) -> QuantizedActivation:
"""SiluAndMul + FP8 static per-tensor quantization."""
d = x.shape[-1] // 2
out_shape = x.shape[:-1] + (d,)
result = torch.empty(out_shape, dtype=FP8_DTYPE, device=x.device)
# TODO(mgoin): read the consumer scale via the contract instead of reaching
# into the kernel-specific input_scale attribute.
scale = linear.input_scale
torch.ops._C.silu_and_mul_quant(result, x, scale)
return QuantizedActivation(
data=result,
scale=scale,
orig_dtype=x.dtype,
orig_shape=out_shape,
quant_key=kFp8StaticTensorSym,
)
def _silu_and_mul_fp8_dynamic_block(
x: torch.Tensor, linear: LinearBase, group_size: int, quant_key: QuantKey
) -> QuantizedActivation:
"""SiluAndMul + FP8 dynamic per-block quantization."""
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
d = x.shape[-1] // 2
out_shape = x.shape[:-1] + (d,)
x = x.reshape(-1, x.shape[-1])
num_tokens = x.shape[0]
num_groups = d // group_size
result = torch.empty((num_tokens, d), dtype=FP8_DTYPE, device=x.device)
scales = torch.empty((num_tokens, num_groups), dtype=torch.float32, device=x.device)
torch.ops._C.silu_and_mul_per_block_quant(
out=result,
input=x,
scales=scales,
group_size=group_size,
scale_ub=None,
is_scale_transposed=False,
)
return QuantizedActivation(
data=result.view(out_shape),
scale=scales.view(out_shape[:-1] + (num_groups,)),
orig_dtype=x.dtype,
orig_shape=out_shape,
quant_key=quant_key,
)
def _silu_and_mul_fp8_dynamic_128(
x: torch.Tensor, linear: LinearBase
) -> QuantizedActivation:
"""SiluAndMul + FP8 dynamic per-group (group=128) quantization."""
return _silu_and_mul_fp8_dynamic_block(x, linear, 128, kFp8Dynamic128Sym)
def _silu_and_mul_nvfp4_dynamic(
x: torch.Tensor, linear: LinearBase
) -> QuantizedActivation:
"""SiluAndMul + NVFP4 dynamic quantization."""
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
d = x.shape[-1] // 2
out_shape = x.shape[:-1] + (d,)
x = x.reshape(-1, x.shape[-1])
num_tokens = x.shape[0]
# NVFP4 packs 2 values into 1 byte
result = torch.empty((num_tokens, d // 2), dtype=FP4_DTYPE, device=x.device)
# Block scale output: swizzled tensor-core layout
# [num_m_tiles, num_k_tiles, 32, 4, 4] of int32-packed FP8 scales, so the
# row/col extents must be padded to the 128x4 tile like
# scaled_fp4_quant's allocator (create_fp4_scale_tensor). Each group of 16
# elements shares one FP8 scale.
rounded_m = round_up(num_tokens, 128)
rounded_n = round_up(d // 16, 4)
block_scale = torch.empty(
(rounded_m, rounded_n // 4), dtype=torch.int32, device=x.device
).view(FP8_DTYPE)
# The kernel folds the global scale into the block scales, and the
# consumer GEMM's alpha (= input_global_scale * weight_global_scale)
# divides it back out, so quantize with the reciprocal like the unfused
# scaled_fp4_quant path does.
input_global_scale_inv = getattr(linear, "input_global_scale_inv", None)
assert input_global_scale_inv is not None, (
"input_global_scale_inv is required for NVFP4 quantization"
)
torch.ops._C.silu_and_mul_nvfp4_quant(
result, block_scale, x, input_global_scale_inv
)
return QuantizedActivation(
data=result.view(out_shape[:-1] + (d // 2,)),
scale=block_scale,
orig_dtype=x.dtype,
orig_shape=out_shape,
quant_key=kNvfp4Dynamic,
)
# (activation module type, consumer input quantization key) -> fused producer.
# Add a row for each supported manual producer.
_FUSED_ACT_QUANT: dict[tuple[type, QuantKey], Callable] = {
(SiluAndMul, kFp8StaticTensorSym): _silu_and_mul_fp8_static,
}
def _relu_squared_static_fp8_quant_supported(
act_fn: torch.nn.Module,
x: torch.Tensor,
linear: LinearBase,
) -> bool:
"""Return whether the ReLU2 static-FP8 producer can consume this input."""
scale = getattr(linear, "input_scale", None)
return (
isinstance(act_fn, ReLUSquaredActivation)
and x.is_cuda
and x.dtype == torch.bfloat16
and x.is_contiguous()
and isinstance(scale, torch.Tensor)
and scale.dtype == torch.float32
and scale.device == x.device
and scale.numel() == 1
)
# Optional per-entry predicates preserve the unconditional fallback contract
# for producers whose supported inputs are narrower than their registry key.
_FUSED_ACT_QUANT_SUPPORT: dict[
tuple[type, QuantKey], Callable[[torch.nn.Module, torch.Tensor, LinearBase], bool]
] = {}
# This path is validated on SM90 and newer CUDA devices.
if current_platform.is_cuda() and current_platform.has_device_capability(90):
_RELU2_STATIC_FP8_KEY = (ReLUSquaredActivation, kFp8StaticTensorSym)
_FUSED_ACT_QUANT[_RELU2_STATIC_FP8_KEY] = relu_squared_static_fp8_quant
_FUSED_ACT_QUANT_SUPPORT[_RELU2_STATIC_FP8_KEY] = (
_relu_squared_static_fp8_quant_supported
)
# Add CUDA-specific entries for dynamic block quantization
if current_platform.is_cuda_alike():
_FUSED_ACT_QUANT[(SiluAndMul, kFp8Dynamic128Sym)] = _silu_and_mul_fp8_dynamic_128
# Add NVFP4 if supported (requires SM100+)
if current_platform.is_cuda() and hasattr(torch.ops._C, "silu_and_mul_nvfp4_quant"):
_FUSED_ACT_QUANT[(SiluAndMul, kNvfp4Dynamic)] = _silu_and_mul_nvfp4_dynamic
def maybe_fused_act_quant(
act_fn: torch.nn.Module,
x: torch.Tensor,
linear: LinearBase,
) -> "torch.Tensor | QuantizedActivation":
"""Apply act_fn, fusing the downstream linear's input quant when possible.
Returns a QuantizedActivation when a fused kernel matches the activation and
the consumer's effective input quantization key, else the plain activation.
"""
key = get_input_quant_key(linear)
if key is not None:
registry_key = (type(act_fn), key)
producer = _FUSED_ACT_QUANT.get(registry_key)
support = _FUSED_ACT_QUANT_SUPPORT.get(registry_key)
if producer is not None and (support is None or support(act_fn, x, linear)):
return producer(x, linear)
return act_fn(x)
@@ -1,14 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointExporter,
)
def kda_prefill_checkpoint_alignment(backend: str) -> int | None:
return 16 if backend == "flashkda" else None
class FlashKDAPrefillCheckpointExporter(MambaPrefillCheckpointExporter):
"""Store FlashKDA recurrent and convolution checkpoint states."""
@@ -1,704 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import torch
import torch.nn.functional as F
from transformers.configuration_utils import PreTrainedConfig
from vllm.config import (
VllmConfig,
get_current_vllm_config,
)
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.mamba.linear.base import LinearAttention
from vllm.model_executor.layers.mamba.linear.minimax_linear_attn import (
MiniMaxText01LinearAttention,
MiniMaxText01LinearKernel,
linear_attention_decode,
)
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
RMSNormGated,
layernorm_fn,
)
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
def _build_rope_parameters(config: PreTrainedConfig) -> dict | None:
rope_parameters = copy.deepcopy(getattr(config, "rope_parameters", None)) or {}
if "rope_theta" not in rope_parameters and hasattr(config, "rope_theta"):
rope_parameters["rope_theta"] = config.rope_theta
if "partial_rotary_factor" not in rope_parameters and hasattr(
config, "partial_rotary_factor"
):
rope_parameters["partial_rotary_factor"] = config.partial_rotary_factor
rope_scaling = getattr(config, "rope_scaling", None)
if isinstance(rope_scaling, dict):
rope_scaling = copy.deepcopy(rope_scaling)
if "type" in rope_scaling and "rope_type" not in rope_scaling:
rope_scaling["rope_type"] = rope_scaling.pop("type")
rope_parameters.update(rope_scaling)
return rope_parameters or None
def clear_linear_attention_cache_for_new_sequences(
kv_cache: torch.Tensor,
state_indices_tensor: torch.Tensor,
attn_metadata: LinearAttentionMetadata,
) -> None:
num_prefills = getattr(attn_metadata, "num_prefills", 0)
if num_prefills <= 0:
return
num_decodes = getattr(attn_metadata, "num_decodes", 0)
prefill_state_indices = getattr(attn_metadata, "state_indices_tensor_p", None)
for prefill_idx in range(num_prefills):
if num_decodes + prefill_idx + 1 >= len(attn_metadata.query_start_loc):
break
q_start = attn_metadata.query_start_loc[num_decodes + prefill_idx]
q_end = attn_metadata.query_start_loc[num_decodes + prefill_idx + 1]
query_len = q_end - q_start
context_len = attn_metadata.seq_lens[num_decodes + prefill_idx] - query_len
if context_len == 0:
if prefill_state_indices is not None:
block_to_clear = prefill_state_indices[prefill_idx]
else:
block_to_clear = state_indices_tensor[num_decodes + prefill_idx]
kv_cache[block_to_clear, ...] = 0
def linear_attention_prefill_and_mix(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_cache: torch.Tensor,
state_indices_tensor: torch.Tensor,
attn_metadata: LinearAttentionMetadata,
slope_rate: torch.Tensor,
block_size: int,
decode_fn,
prefix_fn,
layer_idx: int | None = None,
) -> torch.Tensor:
hidden = []
req_offset = getattr(attn_metadata, "num_decodes", 0)
prefill_state_indices = getattr(attn_metadata, "state_indices_tensor_p", None)
for _prefill_idx in range(getattr(attn_metadata, "num_prefills", 0)):
if req_offset + _prefill_idx + 1 >= len(attn_metadata.query_start_loc):
break
if prefill_state_indices is not None and _prefill_idx >= len(
prefill_state_indices
):
break
if prefill_state_indices is None and _prefill_idx >= len(state_indices_tensor):
break
_start = attn_metadata.query_start_loc[req_offset + _prefill_idx]
_end = attn_metadata.query_start_loc[req_offset + _prefill_idx + 1]
if prefill_state_indices is not None:
slot_id = prefill_state_indices[_prefill_idx]
else:
slot_id = state_indices_tensor[req_offset + _prefill_idx]
qs = q[_start:_end].transpose(0, 1).contiguous()
ks = k[_start:_end].transpose(0, 1).contiguous()
vs = v[_start:_end].transpose(0, 1).contiguous()
slice_layer_cache = kv_cache[slot_id, ...]
out_slice = prefix_fn(
qs,
ks,
vs,
slice_layer_cache,
slope_rate,
block_size,
layer_idx=layer_idx,
)
hidden.append(out_slice.contiguous())
if attn_metadata.num_decode_tokens > 0:
hidden_decode = decode_fn(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
hidden.insert(0, hidden_decode)
if not hidden:
return torch.empty((0, q.size(1) * q.size(2)), device=q.device, dtype=q.dtype)
hidden = torch.concat(hidden, dim=0).contiguous()
return hidden
@triton.jit
def _bailing_linear_attn_decode_spec_step_kernel(
q_ptr,
k_ptr,
v_ptr,
kv_cache_ptr,
slope_rate_ptr,
state_indices_ptr,
query_start_loc_ptr,
num_accepted_tokens_ptr,
output_ptr,
q_start: tl.constexpr,
D: tl.constexpr,
q_b_stride,
q_h_stride,
q_d_stride,
k_b_stride,
k_h_stride,
k_d_stride,
v_b_stride,
v_h_stride,
v_d_stride,
cache_b_stride,
cache_h_stride,
cache_d0_stride,
cache_d1_stride,
state_indices_b_stride,
state_indices_t_stride,
output_b_stride,
output_d_stride,
DRAFT_IDX: tl.constexpr,
STATE_WIDTH: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
req_id = tl.program_id(0)
head_id = tl.program_id(1)
block_id = tl.program_id(2)
req_start = tl.load(query_start_loc_ptr + req_id).to(tl.int64)
req_end = tl.load(query_start_loc_ptr + req_id + 1).to(tl.int64)
query_len = req_end - req_start
if query_len <= DRAFT_IDX:
return
dst_slot = tl.load(
state_indices_ptr
+ req_id * state_indices_b_stride
+ DRAFT_IDX * state_indices_t_stride
).to(tl.int64)
if dst_slot == -1:
return
if DRAFT_IDX == 0:
accepted_offset = tl.load(num_accepted_tokens_ptr + req_id).to(tl.int64) - 1
accepted_offset = tl.maximum(accepted_offset, 0)
accepted_offset = tl.minimum(accepted_offset, STATE_WIDTH - 1)
src_slot = tl.load(
state_indices_ptr
+ req_id * state_indices_b_stride
+ accepted_offset * state_indices_t_stride
).to(tl.int64)
else:
src_slot = tl.load(
state_indices_ptr
+ req_id * state_indices_b_stride
+ (DRAFT_IDX - 1) * state_indices_t_stride
).to(tl.int64)
if src_slot == -1:
return
token_idx = req_start - q_start + DRAFT_IDX
qk_offsets = tl.arange(0, D)
v_offsets = tl.arange(0, BLOCK_SIZE) + block_id * BLOCK_SIZE
qk_mask = qk_offsets < D
v_mask = v_offsets < D
kv_mask = qk_mask[:, None] & v_mask[None, :]
q = tl.load(
q_ptr + token_idx * q_b_stride + head_id * q_h_stride + qk_offsets * q_d_stride,
mask=qk_mask,
other=0.0,
)
k = tl.load(
k_ptr + token_idx * k_b_stride + head_id * k_h_stride + qk_offsets * k_d_stride,
mask=qk_mask,
other=0.0,
)
v = tl.load(
v_ptr + token_idx * v_b_stride + head_id * v_h_stride + v_offsets * v_d_stride,
mask=v_mask,
other=0.0,
)
cache_offsets = (
qk_offsets[:, None] * cache_d0_stride + v_offsets[None, :] * cache_d1_stride
)
src_cache_ptr = (
kv_cache_ptr
+ src_slot * cache_b_stride
+ head_id * cache_h_stride
+ cache_offsets
)
dst_cache_ptr = (
kv_cache_ptr
+ dst_slot * cache_b_stride
+ head_id * cache_h_stride
+ cache_offsets
)
slope = tl.load(slope_rate_ptr + head_id)
decay = tl.exp(-slope)
kv_old = tl.load(src_cache_ptr, mask=kv_mask, other=0.0)
kv_new = k[:, None] * v[None, :] + decay * kv_old
output = tl.sum(q[:, None].to(tl.float32) * kv_new, axis=0)
tl.store(dst_cache_ptr, kv_new, mask=kv_mask)
tl.store(
output_ptr
+ token_idx * output_b_stride
+ (head_id * D + v_offsets) * output_d_stride,
output,
mask=v_mask,
)
def bailing_linear_attention_decode_spec(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_cache: torch.Tensor,
slope_rate: torch.Tensor,
state_indices_tensor: torch.Tensor,
query_start_loc: torch.Tensor,
num_accepted_tokens: torch.Tensor,
q_start: int,
q_end: int | None,
slot_start: int,
slot_end: int | None,
block_size: int,
) -> torch.Tensor:
q_decode = q[q_start:q_end]
k_decode = k[q_start:q_end]
v_decode = v[q_start:q_end]
hidden = torch.empty(
(q_decode.shape[0], q.shape[1] * q.shape[2]),
device=q.device,
dtype=q.dtype,
)
hidden.zero_()
state_indices_tensor = state_indices_tensor[slot_start:slot_end]
query_start_loc = query_start_loc.to(device=q.device)
batch_size = state_indices_tensor.shape[0]
num_heads = q_decode.shape[1]
head_dim = q_decode.shape[2]
assert k_decode.shape == (q_decode.shape[0], num_heads, head_dim)
assert v_decode.shape == (q_decode.shape[0], num_heads, head_dim)
state_width = state_indices_tensor.shape[1]
grid = (batch_size, num_heads, triton.cdiv(head_dim, block_size))
for draft_idx in range(state_width):
_bailing_linear_attn_decode_spec_step_kernel[grid](
q_decode,
k_decode,
v_decode,
kv_cache,
slope_rate,
state_indices_tensor,
query_start_loc,
num_accepted_tokens[:batch_size],
hidden,
q_start,
head_dim,
q_decode.stride(0),
q_decode.stride(1),
q_decode.stride(2),
k_decode.stride(0),
k_decode.stride(1),
k_decode.stride(2),
v_decode.stride(0),
v_decode.stride(1),
v_decode.stride(2),
kv_cache.stride(0),
kv_cache.stride(1),
kv_cache.stride(2),
kv_cache.stride(3),
state_indices_tensor.stride(0),
state_indices_tensor.stride(1),
hidden.stride(0),
hidden.stride(1),
DRAFT_IDX=draft_idx,
STATE_WIDTH=state_width,
BLOCK_SIZE=block_size,
)
return hidden
class BailingGroupRMSNormGate(RMSNormGated):
def __init__(
self,
hidden_size,
eps=1e-5,
group_size=None,
norm_before_gate=True,
device=None,
dtype=None,
):
super().__init__(
hidden_size,
eps=eps,
group_size=group_size,
norm_before_gate=norm_before_gate,
device=device,
dtype=dtype,
activation="sigmoid",
)
# Add custom weight loader for TP sharding
self.weight.weight_loader = self._weight_loader
@staticmethod
def _weight_loader(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None:
"""Load weight with TP sharding."""
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
shard_size = loaded_weight.shape[0] // tp_size
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
param.data.copy_(loaded_weight[shard].contiguous())
# --8<-- [start:bailing_moe_linear_attention]
@PluggableLayer.register("bailing_moe_linear_attention")
class BailingMoELinearAttention(LinearAttention):
"""Pluggable Bailing MoE Linear Attention layer which allows OOT backends
to add custom implementations.
This implements the linear attention mechanism from sglang, adapted for
vLLM's v1 engine with MambaBase interface support.
"""
# --8<-- [end:bailing_moe_linear_attention]
def __init__(
self,
config: PreTrainedConfig,
vllm_config: VllmConfig,
prefix: str = "linear_attn",
):
super().__init__(config, vllm_config, prefix)
self.scaling = self.head_dim**-0.5
self.tp_heads = self.num_heads // self.tp_size
self.max_position_embeddings = config.max_position_embeddings
self.rope_theta = getattr(config, "rope_theta", 600000)
self.tp_kv_heads = self.num_heads // self.tp_size
self.q_size_per_rank = self.head_dim * self.tp_heads
self.kv_size_per_rank = self.head_dim * self.tp_kv_heads
self.use_qk_norm = getattr(config, "use_qk_norm", False)
self.linear_backend = "minimax"
self.linear_scale = self.linear_backend == "minimax"
self.linear_rope = getattr(config, "linear_rope", True)
if hasattr(config, "use_linear_silu"):
self.linear_silu = config.use_linear_silu
elif hasattr(config, "linear_silu"):
self.linear_silu = config.linear_silu
else:
self.linear_silu = False
self.query_key_value = QKVParallelLinear(
self.hidden_size,
self.head_dim,
self.num_heads,
self.num_heads, # MHA: kv_heads = num_heads
bias=(config.use_bias or config.use_qkv_bias),
quant_config=self.quant_config,
prefix=f"{prefix}.query_key_value",
)
if self.use_qk_norm:
self.query_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.key_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.g_proj = ColumnParallelLinear(
self.hidden_size,
self.hidden_inner_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.g_proj",
)
self.dense = RowParallelLinear(
self.hidden_inner_size,
self.hidden_size,
bias=config.use_bias,
quant_config=self.quant_config,
prefix=f"{prefix}.dense",
reduce_results=True,
)
self.group_norm_size = getattr(config, "group_norm_size", 1)
self.rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
assert self.tp_size <= self.group_norm_size, (
"tp_size must be <= group_norm_size for local rms norm"
)
assert self.group_norm_size % self.tp_size == 0, (
"group_norm_size must be divisible by tp_size"
)
# When group_norm_size == 1, group_size equals hidden_size // tp_size
self.g_norm = BailingGroupRMSNormGate(
hidden_size=self.hidden_inner_size // self.tp_size,
eps=self.rms_norm_eps,
group_size=(
self.hidden_inner_size // self.group_norm_size
if self.group_norm_size > 1
else self.hidden_inner_size // self.tp_size
),
)
# use fp32 rotary embedding
rope_parameters = _build_rope_parameters(config)
self.rotary_emb = get_rope(
self.head_dim,
max_position=self.max_position_embeddings,
is_neox_style=True,
rope_parameters=rope_parameters or None,
)
# Build slope tensor for linear attention decay
slope_rate = MiniMaxText01LinearAttention._build_slope_tensor(self.num_heads)
if self.num_hidden_layers <= 1:
self.slope_rate = slope_rate * (1 + 1e-5)
else:
self.slope_rate = slope_rate * (
1 - self.layer_idx / (self.num_hidden_layers - 1) + 1e-5
)
self.tp_slope = self.slope_rate[
self.tp_rank * self.tp_heads : (self.tp_rank + 1) * self.tp_heads
].contiguous()
# Register for compilation
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def get_attn_backend(self):
from vllm.v1.attention.backends.linear_attn import (
BailingLinearAttentionBackend,
)
return BailingLinearAttentionBackend
@staticmethod
def weight_direct_load(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
"""Load weight for linear attention layers.
For FP8 quantized parameters, we need to use the weight_loader if available,
as it handles special cases like tensor parallelism sharding.
"""
# Check if param has a weight_loader (for vLLM ModelWeightParameter)
weight_loader = getattr(param, "weight_loader", None)
if weight_loader is not None:
# Use the weight_loader which handles TP sharding and quantization
weight_loader(param, loaded_weight)
else:
# Fall back to direct copy for standard tensors
assert param.size() == loaded_weight.size(), (
f"Shape mismatch: {param.shape} vs {loaded_weight.shape}"
)
param.data.copy_(loaded_weight)
def forward(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
positions: torch.Tensor,
) -> None:
"""Forward method called by torch.ops.vllm.linear_attention."""
torch.ops.vllm.linear_attention(
hidden_states,
output,
positions,
self.prefix,
)
def _forward(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
positions: torch.Tensor,
) -> None:
"""Actual forward implementation."""
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
if attn_metadata is not None:
assert isinstance(attn_metadata, dict)
attn_metadata = attn_metadata.get(self.prefix) # type: ignore
if attn_metadata is not None:
assert isinstance(attn_metadata, LinearAttentionMetadata)
num_actual_tokens = (
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
)
else:
num_actual_tokens = hidden_states.shape[0]
# QKV projection
qkv, _ = self.query_key_value(hidden_states[:num_actual_tokens])
# use rotary_emb support fp32
qkv = qkv.to(torch.float32)
if self.linear_silu:
qkv = F.silu(qkv)
# Split q, k, v
q, k, v = torch.split(
qkv,
[self.q_size_per_rank, self.kv_size_per_rank, self.kv_size_per_rank],
dim=-1,
)
# Apply QK norm if needed
if self.use_qk_norm:
q = q.reshape(-1, self.tp_heads, self.head_dim)
k = k.reshape(-1, self.tp_kv_heads, self.head_dim)
q = layernorm_fn(
q,
self.query_layernorm.weight.data,
bias=None,
eps=self.rms_norm_eps,
is_rms_norm=True,
)
k = layernorm_fn(
k,
self.key_layernorm.weight.data,
bias=None,
eps=self.rms_norm_eps,
is_rms_norm=True,
)
q = q.reshape(-1, self.q_size_per_rank)
k = k.reshape(-1, self.kv_size_per_rank)
# Apply rotary embeddings
if self.linear_rope:
q, k = self.rotary_emb(positions[:num_actual_tokens], q, k)
# Reshape to [batch, heads, seq_len, head_dim]
q = q.view((qkv.shape[0], self.tp_heads, self.head_dim))
k = k.view((qkv.shape[0], self.tp_kv_heads, self.head_dim))
v = v.view((qkv.shape[0], self.tp_kv_heads, self.head_dim))
# Apply scaling if using minimax backend
if self.linear_scale:
q = q * self.scaling
# Get KV cache and state indices
if attn_metadata is not None:
kv_cache = self.kv_cache[0]
state_indices_tensor = attn_metadata.state_indices_tensor
clear_linear_attention_cache_for_new_sequences(
kv_cache, state_indices_tensor, attn_metadata
)
# Compute attention
decode_only = getattr(attn_metadata, "num_prefills", 0) == 0
if attn_metadata is None:
hidden = torch.empty(
(q.shape[0], q.shape[1] * q.shape[2]), device=q.device, dtype=q.dtype
)
else:
if not decode_only:
hidden = self._prefill_and_mix_infer(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
else:
hidden = self._decode_infer(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
# Apply group norm and gate (matching SGLang behavior)
gate, _ = self.g_proj(hidden_states[:num_actual_tokens])
if self.group_norm_size > 1:
hidden = self.g_norm(hidden, gate)
else:
hidden = self.g_norm(hidden)
hidden = F.sigmoid(gate) * hidden
hidden = hidden.to(hidden_states.dtype)
# Output projection
dense_out, _ = self.dense(hidden)
output[:num_actual_tokens] = dense_out
def _prefill_and_mix_infer(
self, q, k, v, kv_cache, state_indices_tensor, attn_metadata
):
"""Handle prefill (mixed with decode if any)."""
return linear_attention_prefill_and_mix(
q=q,
k=k,
v=v,
kv_cache=kv_cache,
state_indices_tensor=state_indices_tensor,
attn_metadata=attn_metadata,
slope_rate=self.tp_slope,
block_size=self.BLOCK,
decode_fn=self._decode_infer,
prefix_fn=MiniMaxText01LinearKernel.jit_linear_forward_prefix,
layer_idx=self.layer_idx,
)
def _decode_infer(self, q, k, v, kv_cache, state_indices_tensor, attn_metadata):
"""Handle decode (single token per sequence)."""
decode_state_indices = getattr(attn_metadata, "state_indices_tensor_d", None)
num_accepted_tokens = getattr(attn_metadata, "num_accepted_tokens", None)
query_start_loc = getattr(attn_metadata, "query_start_loc_d", None)
if (
decode_state_indices is not None
and decode_state_indices.dim() > 1
and num_accepted_tokens is not None
and query_start_loc is not None
):
return bailing_linear_attention_decode_spec(
q,
k,
v,
kv_cache,
self.tp_slope,
decode_state_indices,
query_start_loc,
num_accepted_tokens,
q_start=0,
q_end=attn_metadata.num_decode_tokens,
slot_start=0,
slot_end=attn_metadata.num_decodes,
block_size=32,
)
decode_state_indices = (
state_indices_tensor
if decode_state_indices is None
else decode_state_indices
)
hidden = linear_attention_decode(
q,
k,
v,
kv_cache,
self.tp_slope,
decode_state_indices,
q_start=0,
q_end=attn_metadata.num_decode_tokens,
slot_start=0,
slot_end=attn_metadata.num_decodes,
block_size=32,
)
return hidden
@@ -1,66 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from transformers import PreTrainedConfig
from vllm.config import (
VllmConfig,
)
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.mamba.abstract import MambaBase
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateDtypeCalculator,
MambaStateShapeCalculator,
)
from vllm.model_executor.models.utils import extract_layer_index
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
class LinearAttention(PluggableLayer, MambaBase):
"""Base class for Linear attention layer."""
def __init__(
self, config: PreTrainedConfig, vllm_config: VllmConfig, prefix: str = ""
):
super().__init__()
self.layer_idx = extract_layer_index(prefix)
self.prefix = prefix
self.model_config = vllm_config.model_config
self.cache_config = vllm_config.cache_config
self.quant_config = vllm_config.quant_config
self.BLOCK = getattr(config, "block", 256)
self.hidden_size = config.hidden_size
self.num_heads = config.num_attention_heads
self.num_hidden_layers = config.num_hidden_layers
self.head_dim = (
config.head_dim
if hasattr(config, "head_dim")
else config.hidden_size // self.num_heads
)
self.hidden_inner_size = self.head_dim * self.num_heads
self.tp_size = get_tensor_model_parallel_world_size()
self.tp_rank = get_tensor_model_parallel_rank()
assert self.num_heads % self.tp_size == 0
@property
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.LINEAR
def get_state_dtype(self) -> tuple[torch.dtype]:
assert self.model_config is not None
assert self.cache_config is not None
return MambaStateDtypeCalculator.linear_attention_state_dtype(
self.model_config.dtype,
self.cache_config.mamba_cache_dtype,
)
def get_state_shape(self) -> tuple[tuple[int, int, int], ...]:
return MambaStateShapeCalculator.linear_attention_state_shape(
num_heads=self.num_heads, tp_size=self.tp_size, head_dim=self.head_dim
)
@@ -1,345 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Callable
import torch
import torch.nn.functional as F
from einops import rearrange
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.lightning_attn import (
lightning_attention,
linear_decode_forward_triton,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.mamba.linear.base import LinearAttention
from vllm.model_executor.layers.minimax_rms_norm import MiniMaxText01RMSNormTP
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
def clear_linear_attention_cache_for_new_sequences(
kv_cache: torch.Tensor,
state_indices_tensor: torch.Tensor,
attn_metadata: LinearAttentionMetadata,
) -> None:
num_prefills = getattr(attn_metadata, "num_prefills", 0)
if num_prefills <= 0:
return
num_decode_tokens = getattr(attn_metadata, "num_decode_tokens", 0)
for prefill_idx in range(num_prefills):
q_start = attn_metadata.query_start_loc[num_decode_tokens + prefill_idx]
q_end = attn_metadata.query_start_loc[num_decode_tokens + prefill_idx + 1]
query_len = q_end - q_start
context_len = (
attn_metadata.seq_lens[num_decode_tokens + prefill_idx] - query_len
)
if context_len == 0:
block_to_clear = state_indices_tensor[num_decode_tokens + prefill_idx]
kv_cache[block_to_clear, ...] = 0
def linear_attention_decode(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_cache: torch.Tensor,
slope_rate: torch.Tensor,
state_indices_tensor: torch.Tensor,
q_start: int = 0,
q_end: int | None = None,
slot_start: int = 0,
slot_end: int | None = None,
block_size: int = 32,
) -> torch.Tensor:
q = q[q_start:q_end].unsqueeze(2).contiguous()
k = k[q_start:q_end].unsqueeze(2).contiguous()
v = v[q_start:q_end].unsqueeze(2).contiguous()
slot_id = state_indices_tensor[slot_start:slot_end]
return linear_decode_forward_triton(
q, k, v, kv_cache, slope_rate, slot_id, block_size
)
def linear_attention_prefill_and_mix(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_cache: torch.Tensor,
state_indices_tensor: torch.Tensor,
attn_metadata: LinearAttentionMetadata,
slope_rate: torch.Tensor,
block_size: int,
decode_fn: Callable[..., torch.Tensor],
prefix_fn: Callable[..., torch.Tensor],
layer_idx: int | None = None,
) -> torch.Tensor:
hidden = []
for _prefill_idx in range(getattr(attn_metadata, "num_prefills", 0)):
if _prefill_idx >= len(attn_metadata.query_start_loc):
break
if _prefill_idx >= len(state_indices_tensor):
break
offset = attn_metadata.num_decode_tokens
_start = attn_metadata.query_start_loc[offset + _prefill_idx]
_end = attn_metadata.query_start_loc[offset + _prefill_idx + 1]
slot_id = state_indices_tensor[offset + _prefill_idx]
qs = q[_start:_end].transpose(0, 1).contiguous()
ks = k[_start:_end].transpose(0, 1).contiguous()
vs = v[_start:_end].transpose(0, 1).contiguous()
slice_layer_cache = kv_cache[slot_id, ...]
out_slice = prefix_fn(
qs,
ks,
vs,
slice_layer_cache,
slope_rate,
block_size,
layer_idx=layer_idx,
)
hidden.append(out_slice.contiguous())
if attn_metadata.num_decode_tokens > 0:
hidden_decode = decode_fn(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
hidden.insert(0, hidden_decode)
if not hidden:
return torch.empty((0, q.size(-1)), device=q.device, dtype=q.dtype)
hidden = torch.concat(hidden, dim=0).contiguous()
return hidden
class MiniMaxText01LinearKernel:
@staticmethod
def jit_linear_forward_prefix(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_caches: torch.Tensor,
slope_rate: torch.Tensor,
block_size: int,
layer_idx: int | None = None,
**kwargs,
) -> torch.Tensor:
slope_rate = slope_rate.to(torch.float32)
should_pad_dim = q.dim() == 3
if should_pad_dim:
q = q.unsqueeze(0)
k = k.unsqueeze(0)
v = v.unsqueeze(0)
b, h, n, d = q.shape
e = d
kv_history = kv_caches.reshape(1, h, d, e).contiguous()
output, kv_history = lightning_attention(
q, k, v, slope_rate, block_size=block_size, kv_history=kv_history
)
kv_caches.copy_(kv_history[:, :, -1, :, :].reshape(h, d, e))
assert output.shape[0] == 1, "batch size must be 1"
return rearrange(output.squeeze(0), "h n d -> n (h d)")
@PluggableLayer.register("minimax_text_01_attention")
class MiniMaxText01LinearAttention(LinearAttention):
def __init__(
self,
config,
vllm_config,
prefix: str = "linear_attn",
) -> None:
super().__init__(config, vllm_config, prefix)
self.tp_heads = self.num_heads // self.tp_size
self.qkv_size = self.num_heads * self.head_dim
self.tp_hidden = self.head_dim * self.tp_heads
self.qkv_proj = ColumnParallelLinear(
self.hidden_size,
self.hidden_inner_size * 3,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.output_gate = ColumnParallelLinear(
self.hidden_size,
self.hidden_inner_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.output_gate",
)
self.out_proj = RowParallelLinear(
self.hidden_inner_size,
self.hidden_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.out_proj",
)
self.norm = MiniMaxText01RMSNormTP(
self.hidden_inner_size,
eps=1e-5,
)
slope_rate = MiniMaxText01LinearAttention._build_slope_tensor(self.num_heads)
if self.num_hidden_layers <= 1:
self.slope_rate = slope_rate * (1 + 1e-5)
else:
self.slope_rate = slope_rate * (
1 - self.layer_idx / (self.num_hidden_layers - 1) + 1e-5
)
self.tp_slope = self.slope_rate[
self.tp_rank * self.tp_heads : (self.tp_rank + 1) * self.tp_heads
].contiguous()
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
@staticmethod
def weight_direct_load(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
assert param.size() == loaded_weight.size()
param.data.copy_(loaded_weight)
return
@staticmethod
def _build_slope_tensor(n_attention_heads: int):
def get_slopes(n):
def get_slopes_power_of_2(n):
start = 2 ** (-(2 ** -(math.log2(n) - 3)))
ratio = start
return [start * ratio**i for i in range(n)]
if math.log2(n).is_integer():
return get_slopes_power_of_2(n)
else:
closest_power_of_2 = 2 ** math.floor(math.log2(n))
return (
get_slopes_power_of_2(closest_power_of_2)
+ get_slopes(2 * closest_power_of_2)[0::2][: n - closest_power_of_2]
)
slopes = torch.tensor(
get_slopes(n_attention_heads), dtype=torch.float32
).reshape(n_attention_heads, 1, 1)
return slopes
def _prefill_and_mix_infer(
self, q, k, v, kv_cache, state_indices_tensor, attn_metadata
):
return linear_attention_prefill_and_mix(
q=q,
k=k,
v=v,
kv_cache=kv_cache,
state_indices_tensor=state_indices_tensor,
attn_metadata=attn_metadata,
slope_rate=self.tp_slope,
block_size=self.BLOCK,
decode_fn=self._decode_infer,
prefix_fn=MiniMaxText01LinearKernel.jit_linear_forward_prefix,
layer_idx=self.layer_idx,
)
def _decode_infer(self, q, k, v, kv_cache, state_indices_tensor, attn_metadata):
hidden = linear_attention_decode(
q,
k,
v,
kv_cache,
self.tp_slope,
state_indices_tensor,
q_start=0,
q_end=attn_metadata.num_decode_tokens,
slot_start=0,
slot_end=attn_metadata.num_decodes,
block_size=32,
)
return hidden
def forward(
self, hidden_states: torch.Tensor, output: torch.Tensor, positions: torch.Tensor
) -> None:
torch.ops.vllm.linear_attention(
hidden_states,
output,
positions,
self.prefix,
)
def _forward(
self, hidden_states: torch.Tensor, output: torch.Tensor, positions: torch.Tensor
) -> None:
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, LinearAttentionMetadata)
num_actual_tokens = (
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
)
else:
num_actual_tokens = hidden_states.shape[0]
qkv, _ = self.qkv_proj(hidden_states[:num_actual_tokens])
qkv32 = qkv.to(torch.float32)
qkvact = torch.nn.functional.silu(qkv32)
qkvact = qkvact.view((qkv.shape[0], self.tp_heads, -1))
q, k, v = torch.split(qkvact, [self.head_dim] * 3, dim=-1)
if attn_metadata is not None:
kv_cache = self.kv_cache[0]
state_indices_tensor = attn_metadata.state_indices_tensor
clear_linear_attention_cache_for_new_sequences(
kv_cache, state_indices_tensor, attn_metadata
)
decode_only = getattr(attn_metadata, "num_prefills", 0) == 0
if attn_metadata is None:
hidden = torch.empty(
(q.shape[0], q.shape[1] * q.shape[2]), device=q.device, dtype=q.dtype
)
else:
if not decode_only:
hidden = self._prefill_and_mix_infer(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
else:
hidden = self._decode_infer(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
hidden = self.norm._forward(hidden)
gate, _ = self.output_gate(hidden_states[:num_actual_tokens])
hidden = F.sigmoid(gate) * hidden
hidden = hidden.to(hidden_states.dtype)
output[:num_actual_tokens], _ = self.out_proj(hidden)
@eager_break_during_capture
def linear_attention(
hidden_states: torch.Tensor,
output: torch.Tensor,
positions: torch.Tensor,
layer_name: str,
) -> None:
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._forward(hidden_states=hidden_states, output=output, positions=positions)
direct_register_custom_op(
op_name="linear_attention",
op_func=linear_attention,
mutates_args=["output"],
)
@@ -1,485 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import NamedTuple
import torch
from torch import nn
from torch.nn.parameter import Parameter
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.mamba.abstract import MambaBase
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateDtypeCalculator,
MambaStateShapeCalculator,
is_conv_state_dim_first,
)
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_scan_fn
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import selective_state_update
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
# --8<-- [start:mamba_mixer]
@PluggableLayer.register("mamba_mixer")
class MambaMixer(MambaBase, PluggableLayer):
"""Compute ∆, A, B, C, and D the state space parameters and compute
the `contextualized_states`. A, D are input independent
(see Mamba paper [1] Section 3.5.2 "Interpretation of A"
for why A isn't selective) ∆, B, C are input-dependent
(this is a key difference between Mamba and the linear time
invariant S4, and is why Mamba is called
**selective** state spaces)
"""
# --8<-- [end:mamba_mixer]
def __init__(
self,
hidden_size: int,
ssm_state_size: int,
conv_kernel_size: int,
intermediate_size: int,
time_step_rank: int,
use_conv_bias: bool,
use_bias: bool,
use_rms_norm: bool,
rms_norm_has_weight: bool = True,
rms_norm_eps: float = 1e-5,
activation="silu",
is_lora_enabled: bool = False,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
):
super().__init__()
self.time_step_rank = time_step_rank
self.ssm_state_size = ssm_state_size
self.use_rms_norm = use_rms_norm
self.activation = activation
self.is_lora_enabled = is_lora_enabled
self.conv_kernel_size = conv_kernel_size
self.intermediate_size = intermediate_size
self.conv1d = ColumnParallelLinear(
input_size=conv_kernel_size,
output_size=intermediate_size,
bias=use_conv_bias,
prefix=f"{prefix}.conv1d",
)
# unsqueeze to fit conv1d weights shape into the linear weights shape.
# Can't do this in `weight_loader` since it already exists in
# `ColumnParallelLinear` and `set_weight_attrs`
# doesn't allow to override it
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
self.in_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=use_bias,
prefix=f"{prefix}.in_proj",
)
# selective projection used to make dt, B and C input dependent
self.x_proj = RowParallelLinear(
intermediate_size,
time_step_rank + ssm_state_size * 2,
bias=False,
prefix=f"{prefix}.x_proj",
)
# time step projection (discretization) -
# In the forward we need to apply dt_proj without the bias,
# as the bias is added in the selective scan kernel.
self.dt_proj = ColumnParallelLinear(
time_step_rank,
intermediate_size,
bias=True,
skip_bias_add=True,
prefix=f"{prefix}.dt_proj",
)
def weight_loader(param: Parameter, loaded_weight: torch.Tensor):
tp_rank = get_tensor_model_parallel_rank()
tp_size = get_tensor_model_parallel_world_size()
param.data.copy_(
loaded_weight.data.split(loaded_weight.shape[0] // tp_size, dim=0)[
tp_rank
]
)
def A_weight_loader(param: Parameter, loaded_weight: torch.Tensor):
weight_loader(param, -torch.exp(loaded_weight.float()))
tp_size = get_tensor_model_parallel_world_size()
self.A = nn.Parameter(
torch.empty(
intermediate_size // tp_size,
ssm_state_size,
dtype=torch.float32,
)
)
self.D = nn.Parameter(torch.ones(intermediate_size // tp_size))
set_weight_attrs(self.D, {"weight_loader": weight_loader})
set_weight_attrs(self.A, {"weight_loader": A_weight_loader})
self.out_proj = RowParallelLinear(
intermediate_size,
hidden_size,
bias=use_bias,
input_is_parallel=True,
prefix=f"{prefix}.out_proj",
)
self.dt_layernorm = (
RMSNorm(
time_step_rank,
eps=rms_norm_eps,
has_weight=rms_norm_has_weight,
)
if use_rms_norm
else None
)
self.b_layernorm = (
RMSNorm(
ssm_state_size,
eps=rms_norm_eps,
has_weight=rms_norm_has_weight,
)
if use_rms_norm
else None
)
self.c_layernorm = (
RMSNorm(
ssm_state_size,
eps=rms_norm_eps,
has_weight=rms_norm_has_weight,
)
if use_rms_norm
else None
)
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
# The inner tuple is (conv_state, ssm_state)
self.kv_cache = (torch.tensor([]), torch.tensor([]))
self.model_config = model_config
self.cache_config = cache_config
self.prefix = prefix
def _ssm_transform(
self, x: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
# LoRA kernel requires contiguous tensor.
# ROCm: Non-contiguous tensors cause incorrect GEMM
# results when batch > 1.
if self.is_lora_enabled or current_platform.is_rocm():
x = x.contiguous()
ssm_params = self.x_proj(x)[0]
time_step, B, C = torch.split(
ssm_params,
[self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
dim=-1,
)
if self.use_rms_norm:
assert self.dt_layernorm is not None
assert self.b_layernorm is not None
assert self.c_layernorm is not None
time_step = self.dt_layernorm(time_step.contiguous())
B = self.b_layernorm(B.contiguous())
C = self.c_layernorm(C.contiguous())
# ROCm: tensor from split is non-contiguous, causing incorrect
# GEMM results in dt_proj.
if current_platform.is_rocm():
time_step = time_step.contiguous()
discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
return discrete_time_step, B, C
def forward(self, hidden_states: torch.Tensor, output: torch.Tensor):
torch.ops.vllm.mamba_mixer(
hidden_states,
output,
_encode_layer_name(self.prefix),
)
def forward_impl(self, hidden_states: torch.Tensor, output: torch.Tensor):
"""Run the Mamba-1 SSM pipeline.
Steps
-----
1. Apply the gated-MLP linear projection to the raw input.
2. Pass the projected sequence through the convolutional mixing layer.
3. Feed the result into the State-Space Model (SSM) blocks.
4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
to produce contextual representations.
5. Project the contextualised sequence back
to the output embedding dimension.
Batch handling
--------------
Prefill and decode tokens are processed by dedicated CUDA
kernels for both the convolutional (conv1d) and SSM stages.
In the case of a mixed batch (containing both prefill and
decode tokens), both sets of kernels are executed independently
and their outputs are concatenated before the final output projection.
"""
forward_context: ForwardContext = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
assert self.cache_config is not None
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, Mamba1AttentionMetadata)
query_start_loc_p = attn_metadata.query_start_loc_p
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
conv_state = (
self.kv_cache[0]
if is_conv_state_dim_first()
else self.kv_cache[0].transpose(-1, -2)
)
ssm_state = self.kv_cache[1]
has_initial_states_p = attn_metadata.has_initial_states_p
cu_chunk_seqlen_p = attn_metadata.cu_chunk_seqlen_p
last_chunk_indices_p = attn_metadata.last_chunk_indices_p
# 1. Gated MLP's linear projection
projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
hidden_states_BC, gate = projected_states.chunk(2, dim=-2)
conv_weights = self.conv1d.weight.view(
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
)
if attn_metadata is None:
# V1 profile run
hidden_states_BC = hidden_states_BC.contiguous()
return self.out_proj(hidden_states_BC.transpose(-2, -1))[0]
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
num_decode_tokens = attn_metadata.num_decode_tokens
has_prefill = num_prefill_tokens > 0
has_decode = num_decode_tokens > 0
num_actual_tokens = num_prefill_tokens + num_decode_tokens
prefill_decode_split = split_batch_to_prefill_and_decode(
hidden_states_BC,
gate,
num_prefill_tokens,
num_decode_tokens,
)
hidden_states_BC_p = prefill_decode_split.hidden_states_BC_p
hidden_states_BC_d = prefill_decode_split.hidden_states_BC_d
gate_p = prefill_decode_split.gate_p
gate_d = prefill_decode_split.gate_d
ssm_outputs = []
if has_prefill:
# 2. Convolution sequence transformation
conv_out_p = causal_conv1d_fn(
hidden_states_BC_p,
conv_weights,
self.conv1d.bias,
activation=self.activation,
conv_states=conv_state,
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
query_start_loc=query_start_loc_p,
metadata=attn_metadata,
)
# 3. State Space Model sequence transformations.
discrete_time_step_p, B_p, C_p = self._ssm_transform(
conv_out_p.transpose(-2, -1)
)
time_proj_bias = self._time_proj_bias()
# 4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
scan_out_p = selective_scan_fn(
conv_out_p,
ssm_state,
discrete_time_step_p,
self.A,
B_p.transpose(-2, -1),
C_p.transpose(-2, -1),
self.D.float(),
gate_p,
time_proj_bias,
delta_softplus=True,
cache_indices=state_indices_tensor_p,
has_initial_state=has_initial_states_p,
query_start_loc=query_start_loc_p,
cu_chunk_seqlen=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
)
ssm_outputs.append(scan_out_p)
if has_decode:
# state_indices_tensor_d is assigned when attn_metadata is not None,
# and has_decode is only True when attn_metadata is not None
assert state_indices_tensor_d is not None
# 2. Convolution sequence transformation
conv_out_d = causal_conv1d_update(
hidden_states_BC_d.transpose(0, 1),
conv_state,
conv_weights,
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices_tensor_d,
).transpose(0, 1)
# 3. State Space Model sequence transformation.
discrete_time_step_d, B_d, C_d = self._ssm_transform(
conv_out_d.transpose(-2, -1)
)
time_proj_bias = self._time_proj_bias()
# 4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
scan_outputs_d = torch.empty_like(hidden_states_BC_d.transpose(0, 1))
selective_state_update(
ssm_state,
conv_out_d.transpose(0, 1),
discrete_time_step_d.transpose(0, 1),
self.A,
B_d,
C_d,
self.D,
time_proj_bias,
z=gate_d.transpose(0, 1),
dt_softplus=True,
state_batch_indices=state_indices_tensor_d,
dst_state_batch_indices=state_indices_tensor_d,
out=scan_outputs_d,
)
scan_outputs_d = scan_outputs_d.transpose(0, 1)
ssm_outputs.insert(0, scan_outputs_d)
scan_outputs_combined = (
ssm_outputs[0] if len(ssm_outputs) == 1 else torch.cat(ssm_outputs, dim=-1)
)
# 5. Final output projection
if self.is_lora_enabled: # Lora kernel requires contiguous tensor.
scan_outputs_combined = scan_outputs_combined.transpose(-2, -1).contiguous()
out = self.out_proj(scan_outputs_combined)[0]
else:
out = self.out_proj(scan_outputs_combined.transpose(-2, -1))[0]
output[:num_actual_tokens] = out
def get_state_dtype(self) -> tuple[torch.dtype]:
assert self.model_config is not None
assert self.cache_config is not None
return MambaStateDtypeCalculator.mamba1_state_dtype(
self.model_config.dtype,
self.cache_config.mamba_cache_dtype,
self.cache_config.mamba_ssm_cache_dtype,
)
def get_state_shape(self) -> tuple[tuple[int, ...], tuple[int, ...]]:
return MambaStateShapeCalculator.mamba1_state_shape(
tp_world_size=get_tensor_model_parallel_world_size(),
intermediate_size=self.intermediate_size,
state_size=self.ssm_state_size,
conv_kernel=self.conv_kernel_size,
)
@property
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.MAMBA1
def _time_proj_bias(self) -> torch.Tensor | None:
if hasattr(self.dt_proj, "bias") and self.dt_proj.bias is not None:
return self.dt_proj.bias.float()
return None
class PrefillDecodeSplit(NamedTuple):
hidden_states_BC_p: torch.Tensor
hidden_states_BC_d: torch.Tensor
gate_p: torch.Tensor
gate_d: torch.Tensor
def split_batch_to_prefill_and_decode(
hidden_states_BC: torch.Tensor,
gate: torch.Tensor,
num_prefill_tokens: int,
num_decode_tokens: int,
) -> PrefillDecodeSplit:
num_actual_tokens = num_prefill_tokens + num_decode_tokens
# In v1, decode tokens come first, then prefill tokens.
hidden_states_BC_d, hidden_states_BC_p = torch.split(
hidden_states_BC[..., :num_actual_tokens],
[num_decode_tokens, num_prefill_tokens],
dim=-1,
)
gate_d, gate_p = torch.split(
gate[..., :num_actual_tokens], [num_decode_tokens, num_prefill_tokens], dim=-1
)
return PrefillDecodeSplit(
hidden_states_BC_p=hidden_states_BC_p,
hidden_states_BC_d=hidden_states_BC_d,
gate_p=gate_p,
gate_d=gate_d,
)
@eager_break_during_capture
def mamba_mixer(
hidden_states: torch.Tensor,
output: torch.Tensor,
layer_name: LayerNameType,
) -> None:
layer_name = _resolve_layer_name(layer_name)
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self.forward_impl(hidden_states=hidden_states, output=output)
direct_register_custom_op(
op_name="mamba_mixer",
op_func=mamba_mixer,
mutates_args=["output"],
)
@@ -1,83 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@triton.jit
def _gather_initial_states_kernel(
state_ptr,
indices_ptr,
has_initial_state_ptr,
output_ptr,
stride_state_batch,
stride_indices,
stride_has_initial_state,
row_size: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
):
block_idx = tl.program_id(0)
batch_idx = tl.program_id(1)
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < row_size
if launch_pdl:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
has_initial_state = tl.load(
has_initial_state_ptr + batch_idx * stride_has_initial_state
).to(tl.int1)
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
state_idx = tl.where(has_initial_state, state_idx, 0)
values = tl.load(
state_ptr + state_idx * stride_state_batch + offsets,
mask=mask & has_initial_state,
other=0.0,
)
tl.store(output_ptr + batch_idx * row_size + offsets, values, mask=mask)
def gather_initial_states(
state: torch.Tensor,
indices: torch.Tensor,
has_initial_state: torch.Tensor,
) -> torch.Tensor:
"""Gather dense state rows, replacing uninitialized rows with zeros."""
assert state.ndim >= 2
assert state.is_cuda or state.is_xpu
assert indices.ndim == 1 and has_initial_state.ndim == 1
assert indices.shape == has_initial_state.shape
assert indices.device == state.device
assert has_initial_state.device == state.device
assert indices.dtype in (torch.int32, torch.int64)
assert has_initial_state.dtype == torch.bool
row_size = state[0].numel()
# Mamba pages may pad stride(0), but each state row remains dense.
assert state[0].is_contiguous()
output = torch.empty(
(indices.numel(), *state.shape[1:]),
dtype=state.dtype,
device=state.device,
)
block_size = min(triton.next_power_of_2(row_size), 1024)
grid = (triton.cdiv(row_size, block_size), indices.numel())
_gather_initial_states_kernel[grid](
state,
indices,
has_initial_state,
output,
state.stride(0),
indices.stride(0),
has_initial_state.stride(0),
row_size=row_size,
BLOCK_SIZE=block_size,
num_warps=8,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return output
@@ -1,105 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup import WarmupIntRange
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
DispatchSpec,
TritonWarmupTensor,
triton_kernel_dispatcher_with_warmup,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@triton.jit
def _scatter_states_kernel(
state_ptr,
src_ptr,
indices_ptr,
stride_state_batch,
stride_src_batch,
stride_indices,
row_size: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
):
block_idx = tl.program_id(0)
batch_idx = tl.program_id(1)
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < row_size
if launch_pdl:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
values = tl.load(src_ptr + batch_idx * stride_src_batch + offsets, mask=mask)
tl.store(state_ptr + state_idx * stride_state_batch + offsets, values, mask=mask)
def _scatter_states_warmup_inputs(
*,
state_shape: tuple[int, ...],
state_dtype: torch.dtype,
indices_dtype: torch.dtype,
max_num_tokens: int,
) -> dict[str, object]:
num_tokens: Any = WarmupIntRange(1, max_num_tokens + 1)
return dict(
state=TritonWarmupTensor(
state_dtype,
shape=(1,) + state_shape,
),
src=TritonWarmupTensor(
state_dtype,
shape=(num_tokens,) + state_shape,
),
indices=TritonWarmupTensor(
indices_dtype,
shape=(num_tokens,),
),
)
@triton_kernel_dispatcher_with_warmup(
kernel=_scatter_states_kernel,
warmup_inputs=_scatter_states_warmup_inputs,
)
def scatter_states(
state: torch.Tensor,
src: torch.Tensor,
indices: torch.Tensor,
) -> DispatchSpec:
"""Scatter ``src`` rows into ``state`` at ``indices`` (in place).
Equivalent to ``state[indices] = src`` but non-atomic and bandwidth-bound,
since mamba cache slots are unique per sequence. ``gather_initial_states``
is the read-side counterpart.
"""
assert state.ndim >= 2
assert state.is_cuda
assert src.ndim == state.ndim
assert indices.ndim == 1
assert indices.device == state.device
assert src.shape[1:] == state.shape[1:]
assert src.shape[0] == indices.shape[0]
assert indices.dtype in (torch.int32, torch.int64)
row_size = state[0].numel()
assert state[0].is_contiguous()
assert src[0].is_contiguous()
block_size = min(triton.next_power_of_2(row_size), 1024)
grid = (triton.cdiv(row_size, block_size), indices.numel())
return grid, dict(
stride_state_batch=state.stride(0),
stride_src_batch=src.stride(0),
stride_indices=indices.stride(0),
row_size=row_size,
BLOCK_SIZE=block_size,
num_warps=8,
launch_pdl=current_platform.is_arch_support_pdl(),
)
File diff suppressed because it is too large Load Diff
@@ -1,5 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .abstract import *
from .common import *
from .special import *
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Set
import torch
import torch.nn as nn
from vllm.tasks import PoolingTask
from vllm.v1.outputs import PoolerOutput
from vllm.v1.pool.metadata import PoolingMetadata
from .common import PoolingParamsUpdate
class Pooler(nn.Module, ABC):
"""The interface required for all poolers used in pooling models in vLLM."""
@abstractmethod
def get_supported_tasks(self) -> Set[PoolingTask]:
"""Determine which pooling tasks are supported."""
raise NotImplementedError
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
"""Construct the updated pooling parameters to use for a supported task."""
return PoolingParamsUpdate()
@abstractmethod
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> PoolerOutput:
raise NotImplementedError
__all__ = ["Pooler"]
@@ -1,158 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Callable
from typing import TypeVar
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import PreTrainedConfig
from vllm.config import ModelConfig
from vllm.logger import init_logger
from vllm.utils.import_utils import resolve_obj_by_qualname
logger = init_logger(__name__)
def get_act_fn(
config: PreTrainedConfig,
static_num_labels: bool = True,
) -> "PoolerActivation":
# get classification act_fn
# Implement alignment with transformers ForSequenceClassificationLoss
# https://github.com/huggingface/transformers/blob/57bb6db6ee4cfaccc45b8d474dfad5a17811ca60/src/transformers/loss/loss_utils.py#L92
num_labels: int | None = None
if static_num_labels:
num_labels = getattr(config, "num_labels", 0)
problem_type = getattr(config, "problem_type", "")
if problem_type == "regression":
return PoolerIdentity()
if problem_type == "single_label_classification":
return PoolerClassify(num_labels=num_labels)
if problem_type == "multi_label_classification":
return PoolerMultiLabelClassify()
# get cross_encoder act_fn
function_name: str | None = None
if (
hasattr(config, "sentence_transformers")
and "activation_fn" in config.sentence_transformers
):
function_name = config.sentence_transformers["activation_fn"]
elif (
hasattr(config, "sbert_ce_default_activation_function")
and config.sbert_ce_default_activation_function is not None
):
function_name = config.sbert_ce_default_activation_function
if function_name is not None:
if not function_name.startswith("torch.nn.modules."):
raise ValueError(
"Loading of activation functions is restricted to "
"torch.nn.modules for security reasons"
)
fn = resolve_obj_by_qualname(function_name)()
return PoolerActivation.wraps(fn)
return PoolerClassify(num_labels=num_labels)
def resolve_classifier_act_fn(
model_config: ModelConfig,
static_num_labels: bool = True,
act_fn: "PoolerActivation | None" = None,
) -> "PoolerActivation":
if act_fn is None:
return get_act_fn(model_config.hf_config, static_num_labels)
if not callable(act_fn):
raise TypeError(f"Expected a callable activation function, got {type(act_fn)}")
return act_fn
_T = TypeVar("_T", torch.Tensor, list[torch.Tensor])
class PoolerActivation(nn.Module, ABC):
@staticmethod
def wraps(module: nn.Module) -> "PoolerActivation":
if isinstance(module, nn.Identity):
return PoolerIdentity()
if isinstance(module, (nn.Sigmoid, nn.Softmax)):
return PoolerClassify()
return LambdaPoolerActivation(module)
@abstractmethod
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
raise NotImplementedError
def forward(self, pooled_data: _T) -> _T:
# shape:
# classify -> (batch_size, num_classes)
# embed -> (batch_size, embedding_size) or list(embedding_size)
if isinstance(pooled_data, list):
return [self.forward_chunk(data) for data in pooled_data]
return self.forward_chunk(pooled_data)
class PoolerIdentity(PoolerActivation):
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
return pooled_data
class PoolerNormalize(PoolerActivation):
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
return F.normalize(pooled_data, p=2, dim=-1)
class PoolerMultiLabelClassify(PoolerActivation):
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
return F.sigmoid(pooled_data)
class PoolerClassify(PoolerActivation):
def __init__(self, *, num_labels: int | None = None) -> None:
super().__init__()
if num_labels == 0:
logger.warning(
"num_labels should be > 0 for classification "
"models, falling back to sigmoid. "
"Please check if the configuration is correct."
)
self.num_labels = num_labels
def extra_repr(self) -> str:
return f"num_labels={self.num_labels}"
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
num_labels = self.num_labels
if num_labels is None:
num_labels = pooled_data.shape[-1]
if num_labels < 2:
return F.sigmoid(pooled_data)
return F.softmax(pooled_data, dim=-1)
class LambdaPoolerActivation(PoolerActivation):
def __init__(self, fn: Callable[[torch.Tensor], torch.Tensor]) -> None:
super().__init__()
self.fn = fn
def extra_repr(self) -> str:
name = getattr(self.fn, "__name__", None)
if name is None:
name = self.fn.__class__.__name__
return f"fn={name}"
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
return self.fn(pooled_data)
@@ -1,32 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from dataclasses import dataclass
from typing import TypeVar
import torch
from vllm.pooling_params import PoolingParams
_T = TypeVar("_T", bound=torch.Tensor | list[torch.Tensor])
ProjectorFn = Callable[[torch.Tensor], torch.Tensor]
ClassifierFn = Callable[[torch.Tensor], torch.Tensor]
ActivationFn = Callable[[_T], _T]
@dataclass(frozen=True)
class PoolingParamsUpdate:
requires_token_ids: bool = False
"""Set this flag to enable CPU prompt token IDs for your pooler."""
def __or__(self, other: "PoolingParamsUpdate") -> "PoolingParamsUpdate":
return PoolingParamsUpdate(
requires_token_ids=self.requires_token_ids or other.requires_token_ids,
)
def apply(self, params: PoolingParams) -> None:
params.requires_token_ids = self.requires_token_ids
__all__ = ["ActivationFn", "ClassifierFn", "ProjectorFn", "PoolingParamsUpdate"]
@@ -1,45 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Poolers that produce an output aggregating all tokens in the sequence."""
from .heads import (
ClassifierPoolerHead,
EmbeddingPoolerHead,
SequencePoolerHead,
SequencePoolerHeadOutput,
)
from .methods import (
CLSPool,
LastPool,
MeanPool,
SequencePoolingMethod,
SequencePoolingMethodOutput,
get_seq_pooling_method,
)
from .poolers import (
SequencePooler,
SequencePoolerOutput,
SequencePoolingFn,
SequencePoolingHeadFn,
pooler_for_classify,
pooler_for_embed,
)
__all__ = [
"SequencePoolerHead",
"SequencePoolerHeadOutput",
"ClassifierPoolerHead",
"EmbeddingPoolerHead",
"SequencePoolingMethod",
"SequencePoolingMethodOutput",
"CLSPool",
"LastPool",
"MeanPool",
"get_seq_pooling_method",
"SequencePooler",
"SequencePoolingFn",
"SequencePoolingHeadFn",
"SequencePoolerOutput",
"pooler_for_classify",
"pooler_for_embed",
]
@@ -1,204 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Set
from typing import TypeAlias
import torch
import torch.nn as nn
from vllm.model_executor.layers.pooler import ActivationFn, ClassifierFn, ProjectorFn
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata
from .methods import SequencePoolingMethodOutput
SequencePoolerHeadOutput: TypeAlias = torch.Tensor | list[torch.Tensor]
class SequencePoolerHead(nn.Module, ABC):
@abstractmethod
def get_supported_tasks(self) -> Set[PoolingTask]:
raise NotImplementedError
@abstractmethod
def forward(
self,
pooled_data: SequencePoolingMethodOutput,
pooling_metadata: PoolingMetadata,
) -> SequencePoolerHeadOutput:
raise NotImplementedError
class EmbeddingPoolerHead(SequencePoolerHead):
def __init__(
self,
projector: ProjectorFn | None = None,
head_dtype: torch.dtype | str | None = None,
activation: ActivationFn | None = None,
) -> None:
super().__init__()
self.projector = projector
self.head_dtype = head_dtype
self.activation = activation
def extra_repr(self) -> str:
attrs = []
if self.head_dtype is not None:
attrs.append(f"head_dtype={self.head_dtype}")
if self.projector is not None:
attrs.append("projector=True")
if self.activation is not None:
attrs.append(f"activation={self.activation.__class__.__name__}")
return ", ".join(attrs)
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"embed"}
def forward(
self,
pooled_data: SequencePoolingMethodOutput,
pooling_metadata: PoolingMetadata,
) -> SequencePoolerHeadOutput:
pooling_params = pooling_metadata.pooling_params
if len(pooled_data) != len(pooling_params):
raise ValueError(
f"pooled_data length ({len(pooled_data)}) does not match "
f"pooling_params length ({len(pooling_params)})"
)
if isinstance(pooled_data, list):
pooled_data = torch.stack(pooled_data)
# pooled_data shape: [batchsize, hidden_size]
if self.head_dtype is not None:
pooled_data = pooled_data.to(self.head_dtype)
# Apply ST projector
if self.projector is not None:
embeddings = self.projector(pooled_data)
else:
embeddings = pooled_data
# embeddings shape: [batchsize, embedding_size]
# for matryoshka representation
dimensions_list = [pooling_param.dimensions for pooling_param in pooling_params]
if any(d is not None for d in dimensions_list):
# change the output dimension
if len(embeddings) != len(dimensions_list):
raise ValueError(
f"embeddings length ({len(embeddings)}) does not match "
f"dimensions_list length ({len(dimensions_list)})"
)
if len(set(dimensions_list)) == 1 and not isinstance(embeddings, list):
# if all dimensions are the same
d = dimensions_list[0]
embeddings = embeddings[..., :d]
else:
embeddings = [
vecs if d is None else vecs[..., :d]
for vecs, d in zip(embeddings, dimensions_list)
]
# for normalize
if self.activation is not None:
flags = [p.use_activation for p in pooling_params]
if len(set(flags)) == 1:
if flags[0]:
embeddings = self.activation(embeddings)
else:
embeddings = [
self.activation(vecs) if f else vecs
for vecs, f in zip(embeddings, flags)
]
# embeddings shape: [batchsize, embedding_size]
return embeddings
class ClassifierPoolerHead(SequencePoolerHead):
def __init__(
self,
classifier: ClassifierFn | None = None,
logit_mean: float | None = None,
logit_sigma: float | None = None,
head_dtype: torch.dtype | str | None = None,
activation: ActivationFn | None = None,
) -> None:
super().__init__()
self.classifier = classifier
self.logit_mean = logit_mean
self.logit_sigma = logit_sigma
self.head_dtype = head_dtype
self.activation = activation
def extra_repr(self) -> str:
attrs = []
if self.head_dtype is not None:
attrs.append(f"head_dtype={self.head_dtype}")
if self.classifier is not None:
attrs.append("classifier=True")
if self.logit_mean is not None:
attrs.append(f"logit_mean={self.logit_mean}")
if self.logit_sigma is not None:
attrs.append(f"logit_sigma={self.logit_sigma}")
if self.activation is not None:
attrs.append(f"activation={self.activation.__class__.__name__}")
return ", ".join(attrs)
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"classify"}
def forward(
self,
pooled_data: SequencePoolingMethodOutput,
pooling_metadata: PoolingMetadata,
) -> SequencePoolerHeadOutput:
pooling_params = pooling_metadata.pooling_params
if len(pooled_data) != len(pooling_params):
raise ValueError(
f"pooled_data length ({len(pooled_data)}) does not match "
f"pooling_params length ({len(pooling_params)})"
)
if isinstance(pooled_data, list):
pooled_data = torch.stack(pooled_data)
# pooled_data shape: [batchsize, hidden_size]
if self.head_dtype is not None:
pooled_data = pooled_data.to(self.head_dtype)
if self.classifier is not None:
logits: SequencePoolerHeadOutput = self.classifier(pooled_data)
else:
logits = pooled_data
# logits shape: [batchsize, num_labels]
# Affine score calibration: activation((logit - mean) / sigma)
if self.logit_mean is not None:
logits = (
[value - self.logit_mean for value in logits]
if isinstance(logits, list)
else logits - self.logit_mean
)
if self.logit_sigma is not None:
logits = (
[value / self.logit_sigma for value in logits]
if isinstance(logits, list)
else logits / self.logit_sigma
)
if self.activation is not None:
flags = [p.use_activation for p in pooling_params]
if len(set(flags)) == 1:
logits = self.activation(logits) if flags[0] else logits
else:
logits = [
self.activation(vecs) if f else vecs
for vecs, f in zip(logits, flags)
]
# logits shape: [batchsize, num_labels]
return logits
@@ -1,119 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Set
from typing import TypeAlias
import torch
import torch.nn as nn
from vllm.config.pooler import SequencePoolingType
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.tasks import PoolingTask
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.pool.metadata import PoolingMetadata
SequencePoolingMethodOutput: TypeAlias = torch.Tensor | list[torch.Tensor]
_MEAN_POOL_ACCUMULATION_CHUNK_BYTES = 16 * 1024 * 1024 # 16MB
class SequencePoolingMethod(nn.Module, ABC):
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"token_embed", "token_classify", "embed", "classify"}
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return PoolingParamsUpdate()
@abstractmethod
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> SequencePoolingMethodOutput:
raise NotImplementedError
class CLSPool(SequencePoolingMethod):
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> SequencePoolingMethodOutput:
pooling_cursor = pooling_metadata.get_pooling_cursor()
if pooling_cursor.is_partial_prefill():
raise RuntimeError("partial prefill is not supported with CLS pooling")
return hidden_states[pooling_cursor.first_token_indices_gpu]
class LastPool(SequencePoolingMethod):
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> SequencePoolingMethodOutput:
pooling_cursor = pooling_metadata.get_pooling_cursor()
return hidden_states[pooling_cursor.last_token_indices_gpu]
class MeanPool(SequencePoolingMethod):
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> SequencePoolingMethodOutput:
pooling_cursor = pooling_metadata.get_pooling_cursor()
if pooling_cursor.is_partial_prefill():
raise RuntimeError("partial prefill is not supported with MEAN pooling")
prompt_lens_cpu = pooling_cursor.prompt_lens_cpu
num_seqs = prompt_lens_cpu.numel()
hidden_size = hidden_states.shape[-1]
if num_seqs == 0:
# early return for empty batch
return hidden_states.new_empty((0, hidden_size), dtype=torch.float32)
prompt_lens = async_tensor_h2d(
prompt_lens_cpu, device=hidden_states.device, dtype=torch.int64
)
# eg. [2, 1, 3] -> [0, 0, 1, 2, 2, 2]
segment_ids = torch.repeat_interleave(
torch.arange(num_seqs, device=hidden_states.device, dtype=torch.long),
prompt_lens,
output_size=int(prompt_lens_cpu.sum()),
)
segment_sums = torch.zeros(
(num_seqs, hidden_size),
dtype=torch.float32,
device=hidden_states.device,
)
bytes_per_token = hidden_size * torch.finfo(torch.float32).bits // 8
chunk_size = max(1, _MEAN_POOL_ACCUMULATION_CHUNK_BYTES // bytes_per_token)
# iterate over the batch in chunks
for start in range(0, hidden_states.shape[0], chunk_size):
end = min(start + chunk_size, hidden_states.shape[0])
# using index_add_ to accumulate for each segment
segment_sums.index_add_(
0,
segment_ids[start:end],
hidden_states[start:end].to(dtype=torch.float32),
)
return segment_sums / prompt_lens.unsqueeze(1)
def get_seq_pooling_method(
pooling_type: SequencePoolingType | str,
) -> SequencePoolingMethod:
if pooling_type == "CLS":
return CLSPool()
if pooling_type == "LAST":
return LastPool()
if pooling_type == "MEAN":
return MeanPool()
raise NotImplementedError(f"Unknown sequence pooling type: {pooling_type!r}")
@@ -1,137 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Set
from typing import TypeAlias
import torch
from vllm.config import PoolerConfig, get_current_vllm_config
from vllm.model_executor.layers.pooler import ClassifierFn, PoolingParamsUpdate
from vllm.model_executor.layers.pooler.abstract import Pooler
from vllm.model_executor.layers.pooler.activations import (
PoolerActivation,
PoolerNormalize,
resolve_classifier_act_fn,
)
from vllm.model_executor.models.adapters import _load_st_projector
from vllm.tasks import POOLING_TASKS, PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata
from .heads import (
ClassifierPoolerHead,
EmbeddingPoolerHead,
SequencePoolerHead,
SequencePoolerHeadOutput,
)
from .methods import (
SequencePoolingMethod,
SequencePoolingMethodOutput,
get_seq_pooling_method,
)
SequencePoolingFn: TypeAlias = Callable[
[torch.Tensor, PoolingMetadata],
SequencePoolingMethodOutput,
]
SequencePoolingHeadFn: TypeAlias = Callable[
[SequencePoolingMethodOutput, PoolingMetadata],
SequencePoolerHeadOutput,
]
SequencePoolerOutput: TypeAlias = torch.Tensor | list[torch.Tensor]
class SequencePooler(Pooler):
"""A layer that pools specific information from hidden states.
This layer does the following:
1. Extracts specific tokens or aggregates data based on pooling method.
2. Postprocesses the output based on pooling head.
3. Returns structured results as `PoolerOutput`.
"""
def __init__(
self,
pooling: SequencePoolingMethod | SequencePoolingFn,
head: SequencePoolerHead | SequencePoolingHeadFn,
) -> None:
super().__init__()
self.pooling = pooling
self.head = head
def extra_repr(self) -> str:
return (
f"pooling={self.pooling.__class__.__name__}, "
f"head={self.head.__class__.__name__}"
)
def get_supported_tasks(self) -> Set[PoolingTask]:
tasks = set(POOLING_TASKS)
if isinstance(self.pooling, SequencePoolingMethod):
tasks &= self.pooling.get_supported_tasks()
if isinstance(self.head, SequencePoolerHead):
tasks &= self.head.get_supported_tasks()
return tasks
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
updates = PoolingParamsUpdate()
if isinstance(self.pooling, SequencePoolingMethod):
updates |= self.pooling.get_pooling_updates(task)
return updates
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> SequencePoolerOutput:
pooled_data = self.pooling(hidden_states, pooling_metadata)
pooled_data = self.head(pooled_data, pooling_metadata)
return pooled_data
def pooler_for_embed(pooler_config: PoolerConfig) -> SequencePooler:
pooling = get_seq_pooling_method(pooler_config.get_seq_pooling_type())
vllm_config = get_current_vllm_config()
model_config = vllm_config.model_config
head = EmbeddingPoolerHead(
head_dtype=model_config.head_dtype,
projector=_load_st_projector(model_config),
activation=PoolerNormalize(),
)
return SequencePooler(pooling=pooling, head=head)
def pooler_for_classify(
pooler_config: PoolerConfig,
*,
pooling: SequencePoolingMethod | SequencePoolingFn | None = None,
classifier: ClassifierFn | None = None,
act_fn: PoolerActivation | None = None,
) -> SequencePooler:
if pooling is None:
pooling = get_seq_pooling_method(pooler_config.get_seq_pooling_type())
vllm_config = get_current_vllm_config()
model_config = vllm_config.model_config
if model_config.pooler_config is None:
raise ValueError(
"model_config.pooler_config must be set for classification pooling"
)
head = ClassifierPoolerHead(
head_dtype=model_config.head_dtype,
classifier=classifier,
logit_mean=model_config.pooler_config.logit_mean,
logit_sigma=model_config.pooler_config.logit_sigma,
activation=resolve_classifier_act_fn(
model_config, static_num_labels=False, act_fn=act_fn
),
)
return SequencePooler(pooling=pooling, head=head)
@@ -1,258 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
from collections.abc import Mapping, Set
from itertools import groupby
import torch
from vllm.config import PoolerConfig
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata
from .abstract import Pooler, PoolerOutput
from .common import ClassifierFn
from .seqwise import (
SequencePoolingFn,
SequencePoolingMethod,
pooler_for_classify,
pooler_for_embed,
)
from .tokwise import pooler_for_token_classify, pooler_for_token_embed
class DispatchPooler(Pooler):
"""Dispatches calls to a sub-pooler based on the pooling task."""
@classmethod
def for_embedding(cls, pooler_config: PoolerConfig):
return cls(
{
"token_embed": pooler_for_token_embed(pooler_config),
"embed": pooler_for_embed(pooler_config),
},
)
@classmethod
def for_seq_cls(
cls,
pooler_config: PoolerConfig,
*,
pooling: SequencePoolingMethod | SequencePoolingFn | None = None,
classifier: ClassifierFn | None = None,
):
return cls(
{
"token_classify": pooler_for_token_classify(
pooler_config,
classifier=classifier,
),
"classify": pooler_for_classify(
pooler_config,
pooling=pooling,
classifier=classifier,
),
}
)
def __init__(self, poolers_by_task: Mapping[PoolingTask, Pooler]) -> None:
super().__init__()
for task, pooler in poolers_by_task.items():
if task not in pooler.get_supported_tasks():
raise ValueError(
f"{pooler=} does not support {task=}. "
f"Supported tasks: {pooler.get_supported_tasks()}"
)
self.poolers_by_task = poolers_by_task
def get_supported_tasks(self) -> Set[PoolingTask]:
return set(self.poolers_by_task)
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return self.poolers_by_task[task].get_pooling_updates(task)
def replace_classifier(
self,
new_classifier: ClassifierFn,
) -> None:
"""Replaces the classifier to the LoRA-wrapped version."""
for task, pooler in self.poolers_by_task.items():
if task not in {"classify"}:
# Now LoRA only supports classification tasks,
# so we only replace the classifier for "classify" task.
continue
head = getattr(pooler, "head", None)
if head is not None and hasattr(head, "classifier"):
head.classifier = new_classifier
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> PoolerOutput:
poolers_by_task = self.poolers_by_task
cursor = pooling_metadata.pooling_cursor
outputs = list[torch.Tensor | None]()
offset = 0
token_offset = 0
for task, group in groupby(pooling_metadata.tasks):
if not (pooler := poolers_by_task.get(task)):
raise ValueError(
f"Unsupported task: {task!r} "
f"Supported tasks: {self.get_supported_tasks()}"
)
num_items = len(list(group))
group_metadata = pooling_metadata[offset : offset + num_items]
if cursor is None:
group_hidden_states = hidden_states
else:
# Slice out this group's tokens so sub-poolers see only their
# portion of the batch. Token offset is computed from the CPU
# `num_scheduled_tokens_cpu` to avoid a GPU->CPU sync.
group_cursor = group_metadata.pooling_cursor
assert group_cursor is not None
num_group_tokens = int(group_cursor.num_scheduled_tokens_cpu.sum())
group_hidden_states = hidden_states[
token_offset : token_offset + num_group_tokens
]
if token_offset:
# Shift first/last indices to be relative to the slice
# so seqwise poolers (which index `hidden_states` directly)
# remain correct.
pooling_cursor = dataclasses.replace(
group_cursor,
first_token_indices_gpu=(
group_cursor.first_token_indices_gpu - token_offset
),
last_token_indices_gpu=(
group_cursor.last_token_indices_gpu - token_offset
),
)
group_metadata = dataclasses.replace(
group_metadata, pooling_cursor=pooling_cursor
)
token_offset += num_group_tokens
group_output: PoolerOutput = pooler(group_hidden_states, group_metadata)
outputs.extend(group_output)
offset += num_items
return outputs
def extra_repr(self) -> str:
s = f"supported_task={self.get_supported_tasks()}"
return s
class IdentityPooler(Pooler):
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"plugin"}
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> PoolerOutput:
return hidden_states
class BOSEOSFilter(Pooler):
"""Filters the BOS and EOS token results from outputs."""
def __init__(
self,
pooler: Pooler,
bos_token_id: int = -1, # -1 disables the filtering
eos_token_id: int = -1,
) -> None:
super().__init__()
self.pooler = pooler
self.bos_token_id = bos_token_id
self.eos_token_id = eos_token_id
def extra_repr(self) -> str:
return f"bos_token_id={self.bos_token_id}, eos_token_id={self.eos_token_id}"
def get_supported_tasks(self) -> Set[PoolingTask]:
return self.pooler.get_supported_tasks()
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return PoolingParamsUpdate(requires_token_ids=True)
def forward(
self,
hidden_states: torch.Tensor | list[torch.Tensor],
pooling_metadata: PoolingMetadata,
) -> PoolerOutput:
pooled_outputs = self.pooler(hidden_states, pooling_metadata)
assert isinstance(pooled_outputs, list)
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
for i, (prompt_len, token_ids) in enumerate(
zip(pooling_metadata.prompt_lens, prompt_token_ids)
):
# Late chunking returns one row per chunk, with special tokens
# already included in the means. Skip BOS/EOS row removal for these
# requests; ordinary token outputs still use the filtering below.
if pooling_metadata.pooling_params[i].late_chunking_params is not None:
continue
pooled_data = pooled_outputs[i]
assert (
isinstance(pooled_data, torch.Tensor)
and pooled_data.shape[0] == prompt_len
)
if int(token_ids[0]) == self.bos_token_id:
pooled_data = pooled_data[1:]
if int(token_ids[-1]) == self.eos_token_id:
pooled_data = pooled_data[:-1]
pooled_outputs[i] = pooled_data.squeeze(-1)
return pooled_outputs
class BgeM3Pooler(Pooler):
def __init__(self, token_classify_pooler: Pooler, embed_pooler: Pooler) -> None:
super().__init__()
self.token_classify_pooler = token_classify_pooler
self.embed_pooler = embed_pooler
def forward(
self, hidden_states: torch.Tensor, pooling_metadata: PoolingMetadata
) -> PoolerOutput:
embed_outputs = self.embed_pooler(hidden_states, pooling_metadata)
token_classify_outputs = self.token_classify_pooler(
hidden_states, pooling_metadata
)
pooler_outputs: list[torch.Tensor] = []
for embed_output, token_classify_output in zip(
embed_outputs, token_classify_outputs
):
pooler_outputs.append(
torch.cat(
[embed_output.view(-1), token_classify_output.view(-1)], dim=-1
)
)
return pooler_outputs
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"embed&token_classify"}
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return self.embed_pooler.get_pooling_updates(
"embed"
) | self.token_classify_pooler.get_pooling_updates("token_classify")
def extra_repr(self) -> str:
s = f"supported_task={self.get_supported_tasks()}"
return s
__all__ = ["BOSEOSFilter", "DispatchPooler", "IdentityPooler", "BgeM3Pooler"]
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Poolers that produce an output for each token in the sequence."""
from .heads import (
TokenClassifierPoolerHead,
TokenEmbeddingPoolerHead,
TokenPoolerHead,
TokenPoolerHeadOutputItem,
)
from .methods import (
AllPool,
StepPool,
TokenPoolingMethod,
TokenPoolingMethodOutputItem,
get_tok_pooling_method,
)
from .poolers import (
TokenPooler,
TokenPoolerOutput,
TokenPoolingFn,
TokenPoolingHeadFn,
pooler_for_token_classify,
pooler_for_token_embed,
)
__all__ = [
"TokenPoolerHead",
"TokenPoolerHeadOutputItem",
"TokenClassifierPoolerHead",
"TokenEmbeddingPoolerHead",
"TokenPoolingMethod",
"TokenPoolingMethodOutputItem",
"AllPool",
"StepPool",
"get_tok_pooling_method",
"TokenPooler",
"TokenPoolerOutput",
"TokenPoolingFn",
"TokenPoolingHeadFn",
"pooler_for_token_classify",
"pooler_for_token_embed",
]
@@ -1,192 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Set
from typing import TypeAlias
import torch
import torch.nn as nn
from vllm.model_executor.layers.pooler import ActivationFn, ClassifierFn, ProjectorFn
from vllm.pooling_params import PoolingParams
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata
from .methods import TokenPoolingMethodOutputItem
TokenPoolerHeadOutputItem: TypeAlias = torch.Tensor | None
def _mean_pool_chunks(data: torch.Tensor, chunk_size: int) -> torch.Tensor:
"""Mean-pool complete contextual states, accumulating in FP32."""
chunk_size = min(chunk_size, len(data))
num_full, remainder = divmod(len(data), chunk_size)
full_end = num_full * chunk_size
full_chunks = data[:full_end].reshape(num_full, chunk_size, data.shape[-1])
if not remainder:
return full_chunks.mean(dim=1, dtype=torch.float32)
return torch.cat(
[
full_chunks.mean(dim=1, dtype=torch.float32),
data[full_end:].mean(dim=0, keepdim=True, dtype=torch.float32),
]
)
class TokenPoolerHead(nn.Module, ABC):
@abstractmethod
def get_supported_tasks(self) -> Set[PoolingTask]:
raise NotImplementedError
@abstractmethod
def forward_chunk(
self,
pooled_data: TokenPoolingMethodOutputItem,
pooling_param: PoolingParams,
) -> TokenPoolerHeadOutputItem:
raise NotImplementedError
def forward(
self,
pooled_data: list[TokenPoolingMethodOutputItem],
pooling_metadata: PoolingMetadata,
) -> list[TokenPoolerHeadOutputItem]:
pooling_params = pooling_metadata.pooling_params
if len(pooled_data) != len(pooling_params):
raise ValueError(
f"pooled_data length ({len(pooled_data)}) does not match "
f"pooling_params length ({len(pooling_params)})"
)
return [self.forward_chunk(d, p) for d, p in zip(pooled_data, pooling_params)]
class TokenEmbeddingPoolerHead(TokenPoolerHead):
def __init__(
self,
head_dtype: torch.dtype | str | None = None,
projector: ProjectorFn | None = None,
activation: ActivationFn | None = None,
) -> None:
super().__init__()
self.head_dtype = head_dtype
self.projector = projector
self.activation = activation
def extra_repr(self) -> str:
attrs = []
if self.head_dtype is not None:
attrs.append(f"head_dtype={self.head_dtype}")
if self.projector is not None:
attrs.append("projector=True")
if self.activation is not None:
attrs.append(f"activation={self.activation.__class__.__name__}")
return ", ".join(attrs)
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"token_embed"}
def forward_chunk(
self,
pooled_data: TokenPoolingMethodOutputItem,
pooling_param: PoolingParams,
) -> TokenPoolerHeadOutputItem:
# for unfinished chunked prefill
if pooled_data is None:
return None
if (late_chunking := pooling_param.late_chunking_params) is not None:
means = _mean_pool_chunks(pooled_data, late_chunking.chunk_size)
pooled_data = (
means.to(pooled_data.dtype) if self.head_dtype is None else means
)
if self.head_dtype is not None:
pooled_data = pooled_data.to(self.head_dtype)
# pooled_data shape: [n_tokens, hidden_size]
# Apply ST projector
if self.projector is not None:
embeddings = self.projector(pooled_data)
else:
embeddings = pooled_data
# embeddings shape: [n_tokens, embedding_size]
# for matryoshka representation
if pooling_param.dimensions is not None:
embeddings = embeddings[..., : pooling_param.dimensions]
# for normalize
if self.activation is not None and pooling_param.use_activation:
embeddings = self.activation(embeddings)
# embeddings shape: [n_tokens, embedding_size]
return embeddings
class TokenClassifierPoolerHead(TokenPoolerHead):
def __init__(
self,
classifier: ClassifierFn | None = None,
logit_mean: float | None = None,
logit_sigma: float | None = None,
head_dtype: torch.dtype | str | None = None,
activation: ActivationFn | None = None,
) -> None:
super().__init__()
self.classifier = classifier
self.logit_mean = logit_mean
self.logit_sigma = logit_sigma
self.head_dtype = head_dtype
self.activation = activation
def extra_repr(self) -> str:
attrs = []
if self.head_dtype is not None:
attrs.append(f"head_dtype={self.head_dtype}")
if self.classifier is not None:
attrs.append("classifier=True")
if self.logit_mean is not None:
attrs.append(f"logit_mean={self.logit_mean}")
if self.logit_sigma is not None:
attrs.append(f"logit_sigma={self.logit_sigma}")
if self.activation is not None:
attrs.append(f"activation={self.activation.__class__.__name__}")
return ", ".join(attrs)
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"token_classify"}
def forward_chunk(
self,
pooled_data: TokenPoolingMethodOutputItem,
pooling_param: PoolingParams,
) -> TokenPoolerHeadOutputItem:
# for unfinished chunked prefill
if pooled_data is None:
return None
if self.head_dtype is not None:
pooled_data = pooled_data.to(self.head_dtype)
# hidden_states shape: [n_token, hidden_size]
if self.classifier is not None:
logits = self.classifier(pooled_data)
else:
logits = pooled_data
# logits shape: [n_token, num_labels]
# Affine score calibration: activation((logit - mean) / sigma)
if self.logit_mean is not None:
logits = logits - self.logit_mean
if self.logit_sigma is not None:
logits = logits / self.logit_sigma
if self.activation is not None and pooling_param.use_activation:
logits = self.activation(logits)
# logits shape: [n_token, num_labels]
return logits
@@ -1,149 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Set
from typing import TypeAlias
import torch
import torch.nn as nn
from vllm.config import get_current_vllm_config
from vllm.config.pooler import TokenPoolingType
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.tasks import PoolingTask
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.pool.metadata import PoolingMetadata
TokenPoolingMethodOutputItem: TypeAlias = torch.Tensor | None
class TokenPoolingMethod(nn.Module, ABC):
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"token_embed", "token_classify"}
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return PoolingParamsUpdate()
@abstractmethod
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> list[TokenPoolingMethodOutputItem]:
raise NotImplementedError
class AllPool(TokenPoolingMethod):
def __init__(self):
super().__init__()
vllm_config = get_current_vllm_config()
scheduler_config = vllm_config.scheduler_config
self.enable_chunked_prefill = scheduler_config.enable_chunked_prefill
# With async scheduling, the next step may overwrite the hidden states
# buffer before outputs are copied to CPU, so they must own their storage.
self.clone_finished = bool(scheduler_config.async_scheduling)
def extra_repr(self) -> str:
return (
f"enable_chunked_prefill={self.enable_chunked_prefill}, "
f"clone_finished={self.clone_finished}"
)
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> list[TokenPoolingMethodOutputItem]:
pooling_cursor = pooling_metadata.get_pooling_cursor()
# Use the already-CPU num_scheduled_tokens tensor so `.tolist()`
# doesn't trigger a GPU->CPU sync. torch.split produces the same
# consecutive slices as indexing with first/last per-sequence indices.
hidden_states_lst = list(
torch.split(hidden_states, pooling_cursor.num_scheduled_tokens_cpu.tolist())
)
if not self.enable_chunked_prefill:
if self.clone_finished:
return [hs.clone() for hs in hidden_states_lst]
return hidden_states_lst
pooling_states = pooling_metadata.pooling_states
finished_mask = pooling_cursor.get_finished_mask()
# If chunked_prefill is enabled
# 1. first store the chunked hidden_states in pooling_states.hidden_states_cache
for p, hs_chunk, finished in zip(
pooling_states, hidden_states_lst, finished_mask
):
# Own data retained across steps or returned without concatenation.
needs_owned_storage = not finished or (
self.clone_finished and not p.hidden_states_cache
)
p.hidden_states_cache.append(
hs_chunk.clone() if needs_owned_storage else hs_chunk
)
# 2. Once prefill is finished, send hidden_states_cache to PoolerHead
output_list = list[TokenPoolingMethodOutputItem]()
for p, finished in zip(pooling_states, finished_mask):
if finished:
hidden_states_cache = p.hidden_states_cache
if len(hidden_states_cache) == 1:
output_list.append(hidden_states_cache[0])
else:
output_list.append(torch.concat(hidden_states_cache, dim=0))
p.clean()
else:
output_list.append(None)
return output_list
class StepPool(AllPool):
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return PoolingParamsUpdate(requires_token_ids=True)
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> list[TokenPoolingMethodOutputItem]:
pooled_data_lst = super().forward(hidden_states, pooling_metadata)
# Use the CPU copy of prompt_token_ids so the step_tag_id mask can be
# resolved to indices without a d2h sync from boolean indexing.
prompt_token_ids_cpu = pooling_metadata.get_prompt_token_ids_cpu()
pooling_params = pooling_metadata.pooling_params
pooled_data = list[torch.Tensor | None]()
for data, token_id_cpu, pooling_param in zip(
pooled_data_lst, prompt_token_ids_cpu, pooling_params
):
# for unfinished chunked prefill
if data is None:
pooled_data.append(None)
else:
step_tag_id = pooling_param.step_tag_id
returned_token_ids = pooling_param.returned_token_ids
if returned_token_ids is not None and len(returned_token_ids) > 0:
data = data[:, returned_token_ids]
if step_tag_id is not None:
idx_cpu = (token_id_cpu == step_tag_id).nonzero(as_tuple=True)[0]
idx = async_tensor_h2d(idx_cpu, data.device)
data = data[idx]
pooled_data.append(data)
return pooled_data
def get_tok_pooling_method(pooling_type: TokenPoolingType | str):
if pooling_type == "ALL":
return AllPool()
if pooling_type == "STEP":
return StepPool()
raise NotImplementedError(f"Unknown tokenwise pooling type: {pooling_type!r}")
@@ -1,144 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Set
from typing import TypeAlias
import torch
from vllm.config import PoolerConfig, get_current_vllm_config
from vllm.model_executor.layers.pooler import (
ClassifierFn,
PoolingParamsUpdate,
ProjectorFn,
)
from vllm.model_executor.layers.pooler.abstract import Pooler
from vllm.model_executor.layers.pooler.activations import (
PoolerActivation,
PoolerNormalize,
resolve_classifier_act_fn,
)
from vllm.model_executor.models.adapters import _load_st_projector
from vllm.tasks import POOLING_TASKS, PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata
from .heads import (
TokenClassifierPoolerHead,
TokenEmbeddingPoolerHead,
TokenPoolerHead,
TokenPoolerHeadOutputItem,
)
from .methods import (
TokenPoolingMethod,
TokenPoolingMethodOutputItem,
get_tok_pooling_method,
)
TokenPoolingFn: TypeAlias = Callable[
[torch.Tensor, PoolingMetadata],
list[TokenPoolingMethodOutputItem],
]
TokenPoolingHeadFn: TypeAlias = Callable[
[list[TokenPoolingMethodOutputItem], PoolingMetadata],
list[TokenPoolerHeadOutputItem],
]
TokenPoolerOutput: TypeAlias = list[torch.Tensor | None]
class TokenPooler(Pooler):
"""A layer that pools specific information from hidden states.
This layer does the following:
1. Extracts specific tokens or aggregates data based on pooling method.
2. Postprocesses the output based on pooling head.
3. Returns structured results as `PoolerOutput`.
"""
def __init__(
self,
pooling: TokenPoolingMethod | TokenPoolingFn,
head: TokenPoolerHead | TokenPoolingHeadFn | None = None,
) -> None:
super().__init__()
self.pooling = pooling
self.head = head
def extra_repr(self) -> str:
head_name = self.head.__class__.__name__ if self.head is not None else None
return f"pooling={self.pooling.__class__.__name__}, head={head_name}"
def get_supported_tasks(self) -> Set[PoolingTask]:
tasks = set(POOLING_TASKS)
if isinstance(self.pooling, TokenPoolingMethod):
tasks &= self.pooling.get_supported_tasks()
if isinstance(self.head, TokenPoolerHead):
tasks &= self.head.get_supported_tasks()
return tasks
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
updates = PoolingParamsUpdate()
if isinstance(self.pooling, TokenPoolingMethod):
updates |= self.pooling.get_pooling_updates(task)
return updates
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> TokenPoolerOutput:
pooled_data = self.pooling(hidden_states, pooling_metadata)
if self.head is not None:
pooled_data = self.head(pooled_data, pooling_metadata)
return pooled_data
def pooler_for_token_embed(
pooler_config: PoolerConfig, projector: ProjectorFn | None = None
) -> TokenPooler:
pooling = get_tok_pooling_method(pooler_config.get_tok_pooling_type())
vllm_config = get_current_vllm_config()
model_config = vllm_config.model_config
head = TokenEmbeddingPoolerHead(
head_dtype=model_config.head_dtype,
projector=projector
if projector is not None
else _load_st_projector(model_config),
activation=PoolerNormalize(),
)
return TokenPooler(pooling=pooling, head=head)
def pooler_for_token_classify(
pooler_config: PoolerConfig,
*,
pooling: TokenPoolingMethod | TokenPoolingFn | None = None,
classifier: ClassifierFn | None = None,
act_fn: PoolerActivation | None = None,
) -> TokenPooler:
if pooling is None:
pooling = get_tok_pooling_method(pooler_config.get_tok_pooling_type())
vllm_config = get_current_vllm_config()
model_config = vllm_config.model_config
if model_config.pooler_config is None:
raise ValueError(
"model_config.pooler_config must be set for token classification pooling"
)
head = TokenClassifierPoolerHead(
head_dtype=model_config.head_dtype,
classifier=classifier,
logit_mean=model_config.pooler_config.logit_mean,
logit_sigma=model_config.pooler_config.logit_sigma,
activation=resolve_classifier_act_fn(
model_config, static_num_labels=False, act_fn=act_fn
),
)
return TokenPooler(pooling=pooling, head=head)
@@ -1,294 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from aiter.ops.shuffle import shuffle_weight
from compressed_tensors.quantization import (
QuantizationArgs,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
int4_w4a16_moe_quant_config,
)
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
CompressedTensorsMoEMethod,
)
from vllm.model_executor.utils import set_weight_attrs
logger = init_logger(__name__)
def _pack_shuffled_int8_to_packed_int4_no_perm(x_shuf_i8: torch.Tensor) -> torch.Tensor:
"""Pack a preshuffled int8 tensor (values in [-8, 7]) into packed int4 bytes.
Each contiguous 8-value block [v0..v7] -> 4 bytes:
b0=(v4<<4)|v0, b1=(v5<<4)|v1, b2=(v6<<4)|v2, b3=(v7<<4)|v3.
This matches the 7-op in-kernel unpack sequence and avoids any v_perm.
"""
flat = x_shuf_i8.contiguous().view(-1).to(torch.int16)
assert flat.numel() % 8 == 0
u = (flat & 0xF).to(torch.uint8).view(-1, 8)
out = torch.empty((u.shape[0], 4), device=u.device, dtype=torch.uint8)
out[:, 0] = u[:, 0] | (u[:, 4] << 4)
out[:, 1] = u[:, 1] | (u[:, 5] << 4)
out[:, 2] = u[:, 2] | (u[:, 6] << 4)
out[:, 3] = u[:, 3] | (u[:, 7] << 4)
return out.view(-1).to(torch.int8)
def _unpack_gptq_int32_to_signed_int4(w_int32):
"""Unpack GPTQ int32 [E, K//8, N] to signed int4 values [E, N, K] (as int8).
Shared by both the packed-int4 and bf16-dequant paths.
"""
E = w_int32.shape[0]
# [E, K//8, N] -> transpose -> [E, N, K//8]
w = w_int32.transpose(1, 2).contiguous()
N = w.shape[1]
K_div8 = w.shape[2]
K = K_div8 * 8
# Unpack int32 -> 8 x uint4 values along K
w_expanded = w.unsqueeze(-1).expand(E, N, K_div8, 8) # [E, N, K//8, 8]
shifts = torch.arange(8, device=w.device) * 4 # [0, 4, 8, ..., 28]
nibbles = ((w_expanded >> shifts) & 0xF).to(torch.int8) # [E, N, K//8, 8]
nibbles = nibbles.reshape(E, N, K) # [E, N, K] unsigned int4 as int8
# Convert unsigned [0,15] to signed [-8,7]
signed = nibbles.to(torch.int16) - 8
signed = signed.to(torch.int8) # [E, N, K] signed int4 as int8
return signed
def _gptq_int32_to_flydsl_packed(w_int32):
"""Convert GPTQ int32 [E, K//8, N] to FlyDSL shuffled packed int4 [E, N, K//2].
Steps:
1. Unpack int32 to individual signed int4 values (as int8)
2. Apply FlyDSL preshuffle (on individual int8 values)
3. Pack with FlyDSL's interleaved int4 packing
"""
signed = _unpack_gptq_int32_to_signed_int4(w_int32)
E, N, K = signed.shape
# FlyDSL preshuffle (operates on individual values)
shuffled = shuffle_weight(signed, layout=(16, 16))
# FlyDSL interleaved int4 packing
packed = _pack_shuffled_int8_to_packed_int4_no_perm(shuffled).contiguous()
return packed.view(E, N, K // 2)
class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
def __init__(
self,
weight_quant: QuantizationArgs,
input_quant: QuantizationArgs | None,
moe: FusedMoEConfig,
layer_name: str | None = None,
):
super().__init__(moe)
self.weight_quant = weight_quant
self.input_quant = input_quant
# Extract properties from weight_quant
assert weight_quant.num_bits == 4
self.num_bits = weight_quant.num_bits
self.packed_factor = 32 // weight_quant.num_bits
self.strategy = weight_quant.strategy
# channelwise is not supported by this kernel
assert weight_quant.strategy == "group"
assert weight_quant.group_size == 32
self.group_size = weight_quant.group_size
assert weight_quant.symmetric, (
"Only symmetric quantization is supported for MoE. "
"Try --moe-backend emulation."
)
def create_weights(
self,
layer: torch.nn.Module,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
self.num_experts = num_experts
self.inter_dim = intermediate_size_per_partition
# Will transpose the loaded weight along the
# intermediate and hidden dim sizes. Will
# shard for TP along the transposed dims
extra_weight_attrs.update(
{"is_transposed": True, "quant_method": self.strategy}
)
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size // self.packed_factor,
w13_num_shards * intermediate_size_per_partition,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_packed", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
w2_weight = torch.nn.Parameter(
torch.empty(
num_experts,
intermediate_size_per_partition // self.packed_factor,
hidden_size,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
w2_scales_size = intermediate_size_per_partition
if self.strategy == "channel":
num_groups_w2 = num_groups_w13 = 1
self.group_size = -1
else:
num_groups_w2 = w2_scales_size // self.group_size
num_groups_w13 = hidden_size // self.group_size
w13_scale = torch.nn.Parameter(
torch.ones(
num_experts,
num_groups_w13,
w13_num_shards * intermediate_size_per_partition,
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_scale)
set_weight_attrs(w13_scale, extra_weight_attrs)
w2_scale = torch.nn.Parameter(
torch.ones(num_experts, num_groups_w2, hidden_size, dtype=params_dtype),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_scale)
set_weight_attrs(w2_scale, extra_weight_attrs)
w2_weight_shape = torch.nn.Parameter(
torch.empty(num_experts, 2), requires_grad=False
)
layer.register_parameter("w2_weight_shape", w2_weight_shape)
set_weight_attrs(w2_weight_shape, extra_weight_attrs)
w13_weight_shape = torch.nn.Parameter(
torch.empty(num_experts, 2), requires_grad=False
)
layer.register_parameter("w13_weight_shape", w13_weight_shape)
set_weight_attrs(w13_weight_shape, extra_weight_attrs)
layer.a13_scale = None
layer.a2_scale = None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Reconfigure packed weights and scales to match flydsl_w4a16 format
# Convert w13 weights
w13 = layer.w13_weight_packed.data
w13 = _gptq_int32_to_flydsl_packed(w13)
w13 = w13.view(-1).contiguous()
layer.w13_weight_packed = torch.nn.Parameter(w13, requires_grad=False)
# Convert w2 weights
w2 = layer.w2_weight_packed.data
w2 = _gptq_int32_to_flydsl_packed(w2)
w2 = w2.view(-1).contiguous()
layer.w2_weight_packed = torch.nn.Parameter(w2, requires_grad=False)
# Convert scales for FlyDSL:
# per-row: [E, 1, N] -> squeeze -> [E, N]
# groupwise: [E, K//gs, N] -> keep as-is (Opt 0: cache-friendly layout)
w13_scale = layer.w13_weight_scale.data
if self.group_size > 0 and w13_scale.dim() == 3 and w13_scale.shape[1] > 1:
E, G, N = w13_scale.shape
w13_scale = (
w13_scale.view(E, G // 2, 2, N)
.permute(0, 1, 3, 2)
.contiguous()
.view(-1)
.contiguous()
)
elif w13_scale.dim() == 3 and w13_scale.shape[1] == 1:
# Per-row: squeeze [E, 1, N] -> [E, N]
w13_scale = w13_scale.squeeze(1)
layer.w13_weight_scale = torch.nn.Parameter(
w13_scale.contiguous(), requires_grad=False
)
w2_scale = layer.w2_weight_scale.data
if self.group_size > 0 and w2_scale.dim() == 3 and w2_scale.shape[1] > 1:
E, G, N = w2_scale.shape
w2_scale = (
w2_scale.view(E, G // 2, 2, N)
.permute(0, 1, 3, 2)
.contiguous()
.view(-1)
.contiguous()
)
elif w2_scale.dim() == 3 and w2_scale.shape[1] == 1:
# Per-row: squeeze [E, 1, N] -> [E, N]
w2_scale = w2_scale.squeeze(1)
layer.w2_weight_scale = torch.nn.Parameter(
w2_scale.contiguous(), requires_grad=False
)
layer.w13_weight_packed.is_shuffled = True
layer.w2_weight_packed.is_shuffled = True
layer.is_aiter_converted = True
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> FusedMoEQuantConfig | None:
assert self.num_bits == 4
return int4_w4a16_moe_quant_config(
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w1_zp=None,
w2_zp=None,
block_shape=[0, self.group_size],
)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
from vllm.model_executor.layers.fused_moe.fused_flydsl_moe import (
fused_flydsl_moe,
)
assert self.moe_quant_config is not None
return fused_flydsl_moe(
x,
layer.w13_weight_packed,
layer.w2_weight_packed,
self.num_experts,
self.inter_dim,
topk_weights,
topk_ids,
w1_scale=self.moe_quant_config.w1_scale,
w2_scale=self.moe_quant_config.w2_scale,
topk=topk_weights.shape[-1],
group_size=self.group_size,
doweight_stage1=layer.apply_router_weight_on_input,
scale_is_bf16=True,
)
@@ -1,272 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
mxfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsMxfp4,
)
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
MarlinExperts,
)
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsMxFp4,
)
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
is_flashinfer_moe_ep_backend,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
B12X_BACKENDS,
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
Mxfp4MoeBackend,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
make_mxfp4_moe_quant_config,
select_mxfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
CompressedTensorsMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_moe_fp4_layer_for_marlin,
)
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
from vllm.platforms import current_platform
logger = init_logger(__name__)
class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
def __init__(self, moe):
super().__init__(moe)
self.group_size = 32
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
self.experts_cls: type[mk.FusedMoEExperts] | None = None
# Backend selection must match the weight preparation below: CUTLASS
# swizzles scales, b12x and XPU consume checkpoint packing, Marlin
# repacks weights and scales, and the megakernels take the checkpoint
# tensors as they are.
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
if moe.moe_backend == "b12x" or is_flashinfer_moe_ep_backend(moe.moe_backend):
self.mxfp4_backend, experts_cls = select_mxfp4_moe_backend(moe)
assert experts_cls is not None
self.experts_cls = experts_cls
self.use_cutlass_mxfp4 = False
elif self.use_cutlass_mxfp4:
logger.info_once("Using CutlassExpertsMxfp4 for MXFP4 MoE")
self.experts_cls = CutlassExpertsMxfp4
elif current_platform.is_xpu():
self.mxfp4_backend = Mxfp4MoeBackend.XPU
self.experts_cls = XPUExpertsMxFp4
logger.info_once("Using XPUExpertsMxFp4 for MXFP4 MoE on XPU platform")
else:
logger.info_once("Using MarlinExperts for MXFP4 MoE")
self.experts_cls = MarlinExperts
@property
def supports_eplb(self) -> bool:
return self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS
def create_weights(
self,
layer: torch.nn.Module,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
layer.num_experts = num_experts
layer.params_dtype = params_dtype
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
self.moe.w13_num_shards * intermediate_size_per_partition,
# 2 fp4 items are packed in the input dimension
hidden_size // 2,
requires_grad=False,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_packed", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
w2_weight = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
# 2 fp4 items are packed in the input dimension
intermediate_size_per_partition // 2,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
w13_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
self.moe.w13_num_shards * intermediate_size_per_partition,
# 2 fp4 items are packed in the input dimension
hidden_size // self.group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
extra_weight_attrs.update(
{"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
)
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
w2_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
# 2 fp4 items are packed in the input dimension
intermediate_size_per_partition // self.group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> FusedMoEQuantConfig | None:
if self.use_cutlass_mxfp4:
# W4A4: both weights and activations quantized to MXFP4
return mxfp4_moe_quant_config(
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
)
else:
# b12x selects W4A8 or W4A16; XPU uses W4A4; Marlin uses W4A16.
return make_mxfp4_moe_quant_config(
mxfp4_backend=self.mxfp4_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
layer=layer,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
layer.w13_weight = torch.nn.Parameter(
layer.w13_weight_packed.data, requires_grad=False
)
delattr(layer, "w13_weight_packed")
layer.w2_weight = torch.nn.Parameter(
layer.w2_weight_packed.data, requires_grad=False
)
delattr(layer, "w2_weight_packed")
if self.use_cutlass_mxfp4:
# Swizzle weight scales from flat checkpoint layout [E, N, K//32]
# to CUTLASS tiled layout [E, numMTiles*numKTiles*512].
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
swizzle_mxfp4_scales,
)
E = layer.w13_weight_scale.shape[0]
w13_N = layer.w13_weight_scale.shape[1]
w13_scale_K = layer.w13_weight_scale.shape[2]
w13_K = w13_scale_K * 32
w2_M = layer.w2_weight_scale.shape[1]
w2_scale_N = layer.w2_weight_scale.shape[2]
w2_N = w2_scale_N * 32
swizzled_w13 = []
swizzled_w2 = []
for e_idx in range(E):
s13 = layer.w13_weight_scale[e_idx]
sw13 = swizzle_mxfp4_scales(s13, w13_N, w13_K)
swizzled_w13.append(sw13.reshape(w13_N, w13_scale_K))
s2 = layer.w2_weight_scale[e_idx]
sw2 = swizzle_mxfp4_scales(s2, w2_M, w2_N)
swizzled_w2.append(sw2.reshape(w2_M, w2_scale_N))
layer.w13_weight_scale = torch.nn.Parameter(
torch.stack(swizzled_w13), requires_grad=False
)
layer.w2_weight_scale = torch.nn.Parameter(
torch.stack(swizzled_w2), requires_grad=False
)
elif self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
w13, w2, w13_scale, w2_scale, _, _ = (
convert_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend=self.mxfp4_backend,
layer=layer,
w13_weight=layer.w13_weight,
w2_weight=layer.w2_weight,
w13_weight_scale=layer.w13_weight_scale,
w2_weight_scale=layer.w2_weight_scale,
)
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
pass
else:
logger.warning_once(
"Your GPU does not have native support for FP4 computation "
"but FP4 quantization is being used. Weight-only FP4 "
"compression will be used leveraging the Marlin kernel. "
"This may degrade performance for compute-heavy workloads."
)
prepare_moe_fp4_layer_for_marlin(layer)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None:
assert self.experts_cls is not None
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
mxfp4_backend=self.mxfp4_backend,
routing_tables=layer._expert_routing_tables(),
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_weight,
layer.w2_weight,
topk_weights,
topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -1,34 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
_PACKED_WEIGHTS = ("w13_weight", "w2_weight")
class CompressedTensorsW4A8Mxfp4MoEMethod(Mxfp4MoEMethod):
"""MXFP4 weights with dynamic FP8 activations in compressed-tensors format.
Only the checkpoint naming and the declared activation scheme differ from
the native MXFP4 path, so backend selection, weight conversion and
execution are inherited.
"""
# The packed -> unpacked rename means a processed state dict no longer
# matches the parameters registered by create_weights.
supports_pre_processed_weights = False
def create_weights(self, layer: RoutedExperts, *args, **kwargs):
super().create_weights(layer, *args, **kwargs)
for name in _PACKED_WEIGHTS:
param = getattr(layer, name)
delattr(layer, name)
layer.register_parameter(f"{name}_packed", param)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
for name in _PACKED_WEIGHTS:
param = getattr(layer, f"{name}_packed")
delattr(layer, f"{name}_packed")
layer.register_parameter(name, param)
super().process_weights_after_loading(layer)
@@ -1,208 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
select_mxfp8_moe_backend,
)
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe import ( # noqa: E501
CompressedTensorsMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
)
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
"""Compressed-tensors MoE method for pre-quantized MXFP8 (W8A8) checkpoints.
Loads FP8 (E4M3) weights with E8M0 uint8 per-group scales (group_size=32)
from checkpoint. Activations are dynamically quantized to MXFP8 at runtime.
Supports FlashInfer TRT-LLM and Marlin backends (auto-selected).
"""
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_block_size = [1, MXFP8_BLOCK_SIZE]
self.fp8_backend, self.experts_cls = select_mxfp8_moe_backend(config=self.moe)
def create_weights(
self,
layer: torch.nn.Module,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
layer.num_experts = num_experts
layer.params_dtype = params_dtype
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
hidden_size,
dtype=MXFP8_VALUE_DTYPE,
),
requires_grad=False,
)
layer.register_parameter("w13_weight", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
w2_weight = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition,
dtype=MXFP8_VALUE_DTYPE,
),
requires_grad=False,
)
layer.register_parameter("w2_weight", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
w13_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
hidden_size // MXFP8_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
extra_weight_attrs.update(
{"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
)
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
w2_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition // MXFP8_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
layer.w13_input_scale = None
layer.w2_input_scale = None
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
layer.weight_block_size = self.weight_block_size
w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format(
fp8_backend=self.fp8_backend,
layer=layer,
w13=layer.w13_weight,
w2=layer.w2_weight,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w13_input_scale=layer.w13_input_scale,
w2_input_scale=layer.w2_input_scale,
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None:
assert self.experts_cls is not None
self.moe_kernel = make_fp8_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> FusedMoEQuantConfig | None:
return make_fp8_moe_quant_config(
fp8_backend=self.fp8_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
a1_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
block_shape=self.weight_block_size,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
layer=layer,
)
def apply_monolithic(
self,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
x,
layer.w13_weight,
layer.w2_weight,
router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_weight,
layer.w2_weight,
topk_weights,
topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -5,8 +5,8 @@ from .compressed_tensors_scheme import CompressedTensorsScheme
from .compressed_tensors_w4a4_mxfp4 import CompressedTensorsW4A4Mxfp4
from .compressed_tensors_w4a4_nvfp4 import CompressedTensorsW4A4Fp4
from .compressed_tensors_w4a8_fp8 import CompressedTensorsW4A8Fp8
from .compressed_tensors_w4a8_int import CompressedTensorsW4A8Int
from .compressed_tensors_w4a8_mxfp4 import CompressedTensorsW4A8Mxfp4
from .compressed_tensors_w4a8_int import CompressedTensorsW4A8Int
from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
from .compressed_tensors_w8a8_mxfp8 import CompressedTensorsW8A8Mxfp8
@@ -23,12 +23,12 @@ __all__ = [
"CompressedTensorsW8A16Fp8",
"CompressedTensorsW8A8Int8",
"CompressedTensorsW8A8Fp8",
"CompressedTensorsW4A4Fp4",
"CompressedTensorsW4A4Mxfp4",
"CompressedTensorsW4A8Mxfp4",
"CompressedTensorsW4A4Fp4",
"CompressedTensorsW8A8Mxfp8",
"CompressedTensorsW4A8Int",
"CompressedTensorsW4A8Fp8",
"CompressedTensorsW8A8Mxfp8",
"CompressedTensorsWNA4Int",
"CompressedTensorsWNA8Int",
]
@@ -1,26 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TurboQuant: KV-cache quantization for vLLM.
Hadamard rotation + per-coordinate Lloyd-Max scalar quantization for
keys, uniform quantization for values.
The core algorithmic pattern implemented for key quantization (Hadamard
rotation followed by deterministic scalar quantization and
re-normalization) was originally established in DRIVE (Vargaftik et al.,
NeurIPS 2021) and EDEN (Vargaftik et al., ICML 2022). This formulation is
also mathematically equivalent to the scalar case of the HIGGS
quantization method (Malinovskii et al., "Pushing the Limits of Large
Language Model Quantization via the Linearity Theorem", NAACL 2025;
preprint arXiv:2411.17525), which subsequently generalized these concepts.
A first application of this approach to KV-cache compression is in "Cache
Me If You Must: Adaptive Key-Value Quantization for Large Language Models"
(Shutova et al., ICML 2025; preprint arXiv:2501.19392). All of these
foundational and application references pre-date the TurboQuant paper
(Zandieh et al., ICLR 2026).
"""
from vllm.model_executor.layers.quantization.turboquant.config import TurboQuantConfig
__all__ = ["TurboQuantConfig"]
@@ -1,87 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Lloyd-Max optimal scalar quantizer for TurboQuant.
After rotating a d-dimensional unit vector by a random orthogonal matrix,
each coordinate approximately follows N(0, 1/d) for d >= 64.
We solve the Lloyd-Max conditions to find optimal centroids.
Based on: turboquant-pytorch/lloyd_max.py (Zandieh et al.)
"""
import math
from functools import lru_cache
import torch
def _gaussian_pdf(x: float, sigma2: float) -> float:
return (1.0 / math.sqrt(2 * math.pi * sigma2)) * math.exp(-x * x / (2 * sigma2))
def _trapz(f, a: float, b: float, n: int = 200) -> float:
"""Trapezoidal numerical integration (replaces scipy.integrate.quad)."""
h = (b - a) / n
result = 0.5 * (f(a) + f(b))
for i in range(1, n):
result += f(a + i * h)
return result * h
def solve_lloyd_max(
d: int,
bits: int,
max_iter: int = 200,
tol: float = 1e-10,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Solve Lloyd-Max optimal quantizer for N(0, 1/d) distribution.
Args:
d: Vector dimension (determines variance = 1/d).
bits: Number of quantization bits.
max_iter: Maximum Lloyd-Max iterations.
tol: Convergence tolerance.
Returns:
centroids: Sorted tensor of 2^bits optimal centroids.
boundaries: Sorted tensor of 2^bits - 1 decision boundaries.
"""
n_levels = 2**bits
sigma2 = 1.0 / d
sigma = math.sqrt(sigma2)
def pdf(x):
return _gaussian_pdf(x, sigma2)
lo, hi = -3.5 * sigma, 3.5 * sigma
centroids = [lo + (hi - lo) * (i + 0.5) / n_levels for i in range(n_levels)]
for _ in range(max_iter):
boundaries = [
(centroids[i] + centroids[i + 1]) / 2.0 for i in range(n_levels - 1)
]
edges = [lo * 3] + boundaries + [hi * 3]
new_centroids = []
for i in range(n_levels):
a, b = edges[i], edges[i + 1]
num = _trapz(lambda x: x * pdf(x), a, b)
den = _trapz(pdf, a, b)
new_centroids.append(num / den if den > 1e-15 else centroids[i])
if max(abs(new_centroids[i] - centroids[i]) for i in range(n_levels)) < tol:
break
centroids = new_centroids
boundaries = [(centroids[i] + centroids[i + 1]) / 2.0 for i in range(n_levels - 1)]
return (
torch.tensor(centroids, dtype=torch.float32),
torch.tensor(boundaries, dtype=torch.float32),
)
@lru_cache(maxsize=32)
def get_centroids(d: int, bits: int) -> torch.Tensor:
"""Get precomputed Lloyd-Max centroids (cached)."""
centroids, _ = solve_lloyd_max(d, bits)
return centroids
@@ -1,261 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TurboQuant configuration."""
from __future__ import annotations
import logging
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from vllm.config import ModelConfig
logger = logging.getLogger(__name__)
# Named TQ presets: each maps to frozen config parameters.
# key_quant_bits: 8 = FP8 keys, 3-4 = MSE (Lloyd-Max) quantized keys.
# value_quant_bits: 3-4 = uniform quantized values.
TQ_PRESETS: dict[str, dict] = {
"turboquant_k8v4": {
"key_quant_bits": 8,
"value_quant_bits": 4,
"norm_correction": False,
},
"turboquant_4bit_nc": {
"key_quant_bits": 4,
"value_quant_bits": 4,
"norm_correction": True,
},
"turboquant_k3v4_nc": {
"key_quant_bits": 3,
"value_quant_bits": 4,
"norm_correction": True,
},
"turboquant_3bit_nc": {
"key_quant_bits": 3,
"value_quant_bits": 3,
"norm_correction": True,
},
}
@dataclass
class TurboQuantConfig:
"""Configuration for TurboQuant KV-cache quantization.
Applies Hadamard rotation followed by per-coordinate Lloyd-Max scalar
quantization for keys, and uniform quantization for values.
Historical note: the core algorithmic pattern implemented for key
quantization (Hadamard rotation followed by deterministic scalar
quantization and re-normalization) was originally established in DRIVE
(Vargaftik et al., NeurIPS 2021) and EDEN (Vargaftik et al., ICML
2022). This formulation is also mathematically equivalent to the
scalar case of the HIGGS quantization method (Malinovskii et al.,
"Pushing the Limits of Large Language Model Quantization via the
Linearity Theorem", NAACL 2025; preprint arXiv:2411.17525), which
subsequently generalized these concepts.
A first application of this approach to KV-cache compression is in
"Cache Me If You Must: Adaptive Key-Value Quantization for Large
Language Models" (Shutova et al., ICML 2025; preprint
arXiv:2501.19392). All of these foundational and application
references pre-date the TurboQuant paper (Zandieh et al., ICLR 2026).
QJL is intentionally omitted: community consensus (5+ independent
groups) found it hurts attention quality by amplifying variance
through softmax.
Named presets (use via --kv-cache-dtype):
turboquant_k8v4: FP8 keys + 4-bit values, 2.6x, +1.17% PPL
turboquant_4bit_nc: 4-bit MSE keys + 4-bit values + NC, 3.8x, +2.71%
turboquant_k3v4_nc: 3-bit MSE keys + 4-bit values + NC, ~3.5x, +10.63%
turboquant_3bit_nc: 3-bit MSE keys + 3-bit values + NC, 4.9x, +20.59%
Args:
head_dim: Attention head dimension (e.g. 64, 96, 128).
key_quant_bits: Bits for key quantization. 8 = FP8 keys (no
rotation/MSE). 3-4 = Lloyd-Max MSE quantized keys.
value_quant_bits: Bits per value dimension for uniform quantization.
3 = 8 levels, 4 = 16 levels (default).
norm_correction: Re-normalize centroid vectors to unit norm before
inverse rotation during dequant. Fixes quantization-induced norm
distortion, improving PPL by ~0.8% at 4-bit.
"""
head_dim: int = 128
key_quant_bits: int = 3 # 3-4 = MSE keys, 8 = FP8 keys
value_quant_bits: int = 4 # 3-4 = uniform quantized values
seed: int = 42 # kept for backward compatibility; no longer used internally
norm_correction: bool = False
@property
def key_fp8(self) -> bool:
"""Whether keys are stored as FP8 — no rotation/quantization needed."""
return self.key_quant_bits == 8
@property
def mse_bits(self) -> int:
"""MSE quantizer bit-width (determines centroid count: 2^mse_bits).
For MSE key modes, equals key_quant_bits.
For FP8 key mode, falls back to value_quant_bits (centroids are still
needed for continuation-prefill dequant and decode kernel params).
"""
if self.key_fp8:
return self.value_quant_bits
return self.key_quant_bits
@property
def key_mse_bits(self) -> int:
"""MSE bits actually used for key quantization (0 if FP8 keys)."""
if self.key_fp8:
return 0
return self.key_quant_bits
@property
def centroid_bits(self) -> int:
"""Bits for centroid generation — always non-zero."""
return self.mse_bits
@property
def n_centroids(self) -> int:
return 2**self.mse_bits
@property
def key_packed_size(self) -> int:
"""Packed bytes for a single KEY vector.
FP8 mode (key_quant_bits=8):
head_dim bytes (1 byte per element, no overhead).
TQ mode:
- MSE indices: ceil(head_dim * key_mse_bits / 8) bytes
- vec_norm: 2 bytes (float16)
"""
if self.key_fp8:
return self.head_dim # 1 byte per element
mse_bytes = math.ceil(self.head_dim * self.key_mse_bits / 8)
norm_bytes = 2 # vec_norm fp16
return mse_bytes + norm_bytes
@property
def effective_value_quant_bits(self) -> int:
"""Actual bits used for value storage."""
return self.value_quant_bits
@property
def value_packed_size(self) -> int:
"""Packed bytes for a single VALUE vector.
Uniform quantization: ceil(head_dim * bits / 8) + 4 bytes (scale + zero fp16).
"""
data_bytes = math.ceil(self.head_dim * self.value_quant_bits / 8)
return data_bytes + 4 # +2 scale(fp16) +2 zero(fp16)
@property
def slot_size(self) -> int:
"""Total packed bytes per head per position (key + value combined).
Layout: [key_packed | value_packed]
"""
return self.key_packed_size + self.value_packed_size
@property
def slot_size_aligned(self) -> int:
"""Slot size rounded up to next even number.
Even-number is required so effective_head_size = slot_size_aligned // 2
is integral.
"""
s = self.slot_size
return s + (s % 2) # round up to even
@staticmethod
def get_boundary_skip_layers(
model_config: ModelConfig,
n: int = 2,
) -> list[str]:
"""Layer indices to skip TQ compression (boundary protection).
For hybrid models (attention + Mamba/linear-attention), boundary
protection is disabled — hybrids typically have only 8-12
full-attention layers and a hard n=2 on each side would cover
~40 % of them. The dense GSM8K baselines that motivate n=2
don't apply to hybrids.
For dense models, skips first N and last N attention layers.
Empirically required for aggressive presets (k3v4_nc, 3bit_nc)
— without it GSM8K drops ~30 points on Qwen3-4B.
"""
if model_config.is_hybrid:
attn_indices = _get_full_attention_layer_indices(model_config)
if not attn_indices:
raise NotImplementedError(
"TurboQuant KV cache requires identifiable "
"full-attention layers, but none were found in "
"the hybrid model config."
)
logger.info("TQ hybrid: full-attention layers %s", attn_indices)
return []
num_layers = model_config.hf_text_config.num_hidden_layers
if n <= 0 or num_layers <= 0:
return []
n = min(n, num_layers // 2) # don't skip more than half
first = list(range(n))
last = list(range(num_layers - n, num_layers))
# Deduplicate (if num_layers <= 2*n)
indices = sorted(set(first + last))
return [str(i) for i in indices]
@staticmethod
def from_cache_dtype(cache_dtype: str, head_dim: int) -> TurboQuantConfig:
"""Create config from a named preset.
Valid presets: turboquant_k8v4, turboquant_4bit_nc, etc.
"""
if cache_dtype not in TQ_PRESETS:
valid = ", ".join(TQ_PRESETS.keys())
raise ValueError(
f"Unknown TurboQuant cache dtype: {cache_dtype!r}. "
f"Valid presets: {valid}"
)
preset = TQ_PRESETS[cache_dtype]
return TurboQuantConfig(
head_dim=head_dim,
key_quant_bits=preset["key_quant_bits"],
value_quant_bits=preset["value_quant_bits"],
norm_correction=preset["norm_correction"],
)
def _get_full_attention_layer_indices(model_config: ModelConfig) -> list[int]:
"""Global indices of full-attention layers in a hybrid model.
Covers the conventions used across vLLM: ``layer_types`` (Qwen3.5/Next),
``layers_block_type`` (Jamba/Zamba2), ``attn_type_list`` (Minimax).
"""
text_cfg = model_config.hf_text_config
hf_cfg = model_config.hf_config
layer_types = getattr(text_cfg, "layer_types", None)
if layer_types is not None:
return [
i for i, t in enumerate(layer_types) if t in ("full_attention", "attention")
]
layers_block_type = getattr(text_cfg, "layers_block_type", None)
if layers_block_type is not None:
return [
i for i, t in enumerate(layers_block_type) if t in ("attention", "hybrid")
]
attn_type_list = getattr(hf_cfg, "attn_type_list", None)
if attn_type_list is not None:
return [i for i, t in enumerate(attn_type_list) if t == 1]
return []
@@ -1,207 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility functions used for tests and benchmarks."""
import numpy as np
import torch
from vllm import _custom_ops as ops
from vllm.scalar_type import ScalarType, scalar_types
from .marlin_utils import GPTQ_MARLIN_TILE, marlin_permute_scales, marlin_zero_points
from .quant_utils import (
get_pack_factor,
gptq_quantize_weights,
quantize_weights,
)
class MarlinWorkspace:
def __init__(self, out_features, min_thread_n, max_parallel):
assert out_features % min_thread_n == 0, (
"out_features = {} is indivisible by min_thread_n = {}".format(
out_features, min_thread_n
)
)
max_workspace_size = (out_features // min_thread_n) * max_parallel
self.scratch = torch.zeros(max_workspace_size, dtype=torch.int, device="cuda")
def marlin_permute_weights(
q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE, is_a_8bit=False
):
assert q_w.shape == (size_k, size_n)
assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
if is_a_8bit:
# Permute weights to 32x32 marlin tiles
q_w = q_w.reshape((size_k // (tile * 2), tile * 2, size_n // tile, tile))
else:
# Permute weights to 16x64 marlin tiles
q_w = q_w.reshape((size_k // tile, tile, size_n // tile, tile))
q_w = q_w.permute((0, 2, 1, 3))
q_w = q_w.reshape((size_k // tile, size_n * tile))
q_w = q_w.reshape((-1, perm.numel()))[:, perm].reshape(q_w.shape)
return q_w
def marlin_weights(q_w, size_k, size_n, num_bits, perm, is_a_8bit=False):
# Permute
q_w = marlin_permute_weights(q_w, size_k, size_n, perm, is_a_8bit=is_a_8bit)
# Pack
pack_factor = get_pack_factor(num_bits)
orig_device = q_w.device
q_w = q_w.cpu().numpy().astype(np.uint32)
q_packed = np.zeros((q_w.shape[0], q_w.shape[1] // pack_factor), dtype=np.uint32)
for i in range(pack_factor):
q_packed |= q_w[:, i::pack_factor] << num_bits * i
q_packed = torch.from_numpy(q_packed.astype(np.int32)).to(orig_device)
return q_packed
def get_weight_perm(num_bits: int, is_a_8bit: bool = False):
perm_list: list[int] = []
if is_a_8bit:
for i in range(32):
perm1 = []
col = i // 4
for block in [0, 1]:
for row in [
4 * (i % 4),
4 * (i % 4) + 1,
4 * (i % 4) + 2,
4 * (i % 4) + 3,
4 * (i % 4 + 4),
4 * (i % 4 + 4) + 1,
4 * (i % 4 + 4) + 2,
4 * (i % 4 + 4) + 3,
]:
perm1.append(16 * row + col + 8 * block)
for j in range(2):
perm_list.extend([p + 512 * j for p in perm1])
else:
for i in range(32):
perm1 = []
col = i // 4
for block in [0, 1]:
for row in [
2 * (i % 4),
2 * (i % 4) + 1,
2 * (i % 4 + 4),
2 * (i % 4 + 4) + 1,
]:
perm1.append(16 * row + col + 8 * block)
for j in range(4):
perm_list.extend([p + 256 * j for p in perm1])
perm = np.array(perm_list)
if num_bits == 4:
if is_a_8bit: # noqa: SIM108
interleave = np.array([0, 4, 1, 5, 2, 6, 3, 7])
else:
interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7])
elif num_bits == 8:
if is_a_8bit: # noqa: SIM108
interleave = np.array([0, 1, 2, 3])
else:
interleave = np.array([0, 2, 1, 3])
else:
raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
perm = torch.from_numpy(perm)
return perm
def marlin_quantize(
w: torch.Tensor,
quant_type: ScalarType,
group_size: int,
input_dtype: torch.dtype | None = None,
):
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
size_k, size_n = w.shape
num_bits = quant_type.size_bits
# Normalize group_size
if group_size == -1:
group_size = size_k
assert group_size <= size_k
w_ref, q_w, s = gptq_quantize_weights(w, quant_type, group_size)
# Reformat to marlin
weight_perm = get_weight_perm(num_bits, is_a_8bit)
marlin_q_w = marlin_weights(
q_w, size_k, size_n, num_bits, weight_perm, is_a_8bit=is_a_8bit
)
marlin_s = marlin_permute_scales(s, size_k, size_n, group_size, is_a_8bit=is_a_8bit)
if input_dtype == torch.float8_e4m3fn and quant_type == scalar_types.uint4b8:
ops.marlin_int4_fp8_preprocess(marlin_q_w, inplace=True)
marlin_s = marlin_s * 512
# Create result
res_list = [w_ref, marlin_q_w, marlin_s]
for i in range(len(res_list)):
res_list[i] = res_list[i].to(w.device)
return res_list
def awq_marlin_quantize(
w: torch.Tensor,
quant_type: ScalarType,
group_size: int,
input_dtype: torch.dtype | None = None,
):
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
size_k, size_n = w.shape
# Normalize group_size
if group_size == -1:
group_size = size_k
assert group_size <= size_k
# Detect num groups
assert size_k % group_size == 0
num_groups = size_k // group_size
# Quantize with zp
w_ref, q_w, s, zp = quantize_weights(w, quant_type, group_size, zero_points=True)
if input_dtype == torch.float8_e4m3fn and quant_type == scalar_types.uint4:
repeated_zp = zp.repeat_interleave(group_size, 0)
q_w_old = q_w
q_w = q_w_old - repeated_zp
q_w[q_w < 0] = 15 - q_w_old[q_w < 0]
s = s * 512
# Reformat to marlin
weight_perm = get_weight_perm(quant_type.size_bits, is_a_8bit)
marlin_q_w = marlin_weights(
q_w, size_k, size_n, quant_type.size_bits, weight_perm, is_a_8bit=is_a_8bit
)
marlin_s = marlin_permute_scales(s, size_k, size_n, group_size, is_a_8bit=is_a_8bit)
marlin_zp = marlin_zero_points(
zp, num_groups, size_n, quant_type.size_bits, is_a_8bit=is_a_8bit
)
# Create result
res_list = [w_ref, marlin_q_w, marlin_s, marlin_zp]
for i in range(len(res_list)):
res_list[i] = res_list[i].to(w.device)
return res_list
-279
View File
@@ -1,279 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py
# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
#
# Copyright 2023 The Qwen team.
# Copyright 2023 The vLLM team.
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Shared resampler perceiver network used in multimodal models and
related helpers for sincos positional embeddings.
Example models: Qwen (Qwen-VL), MiniCPM-V 2.0
"""
import math
from collections.abc import Callable
from functools import partial
import numpy as np
import torch
import torch.nn.functional as F
from torch import nn
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
def get_abs_pos(abs_pos: torch.Tensor, tgt_size: torch.Tensor | int) -> torch.Tensor:
# abs_pos: L, C
# tgt_size: (H, W)
# return: M, C
src_size = int(math.sqrt(abs_pos.size(0)))
dtype = abs_pos.dtype
if isinstance(tgt_size, int):
tgt_size = (tgt_size, tgt_size)
if src_size == tgt_size[0] and src_size == tgt_size[1]:
return abs_pos
return (
F.interpolate(
abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
size=(tgt_size[0], tgt_size[1]),
mode="bicubic",
align_corners=False,
)
.permute(0, 2, 3, 1)
.flatten(0, 2)
.to(dtype=dtype)
)
# sin/cos positional embedding helpers are adapted from:
# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
def get_1d_sincos_pos_embed_from_grid(
embed_dim: int, pos: np.ndarray, version: tuple[int, int] = (2, 0)
) -> torch.Tensor:
"""embed_dim: output dimension for each position
pos: a list of positions to be encoded: size (M,) / (H, W)
out: (M, D) / (H, W, D)
"""
assert embed_dim % 2 == 0
omega = np.arange(embed_dim // 2, dtype=np.float32)
omega /= embed_dim / 2.0
omega = 1.0 / 10000**omega # (D/2,)
if version == (2, 0):
pos = pos.reshape(-1) # (M,)
out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product
emb_sin = np.sin(out) # (M, D/2)
emb_cos = np.cos(out) # (M, D/2)
emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
else:
out = np.einsum("hw,d->hwd", pos, omega) # (H, W, D/2), outer product
emb_sin = np.sin(out) # (H, W, D/2)
emb_cos = np.cos(out) # (H, W, D/2)
emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
return emb
def get_2d_sincos_pos_embed_from_grid(
embed_dim: int, grid: np.ndarray, version: tuple[int, int] = (2, 0)
) -> torch.Tensor:
assert embed_dim % 2 == 0
# use half of dimensions to encode grid_h
emb_h = get_1d_sincos_pos_embed_from_grid(
embed_dim // 2, grid[0], version
) # (H*W, D/2) or (H, W, D/2)
emb_w = get_1d_sincos_pos_embed_from_grid(
embed_dim // 2, grid[1], version
) # (H*W, D/2) or (H, W, D/2)
if version == (2, 0):
emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
else:
emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D)
return emb
def get_2d_sincos_pos_embed(
embed_dim: int,
grid_size: int | tuple[int, int],
cls_token: bool = False,
version: tuple[int, int] = (2, 0),
) -> torch.Tensor:
"""grid_size: int of the grid height and width
return:
pos_embed: [grid_size*grid_size, embed_dim] or
[1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
"""
if isinstance(grid_size, int):
grid_h_size, grid_w_size = grid_size, grid_size
else:
grid_h_size, grid_w_size = grid_size[0], grid_size[1]
grid_h = np.arange(grid_h_size, dtype=np.float32)
grid_w = np.arange(grid_w_size, dtype=np.float32)
grid = np.meshgrid(grid_w, grid_h) # here w goes first
grid = np.stack(grid, axis=0)
assert isinstance(grid, np.ndarray) and grid.shape == (2, grid_h_size, grid_w_size)
if version == (2, 0):
grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
if cls_token:
pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
else:
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
return pos_embed
class BaseResampler(nn.Module):
"""A 2D perceiver-resampler network with one cross attention layers by
(grid_size**2) learnable queries and 2d sincos pos_emb.
Outputs:
A tensor with the shape of (grid_size**2, embed_dim)
"""
def __init__(
self,
num_queries: int,
embed_dim: int,
num_heads: int,
kv_dim: int | None = None,
norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
do_post_projection: bool = True,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.num_queries = num_queries
self.embed_dim = embed_dim
self.num_heads = num_heads
self.query = nn.Parameter(torch.empty(self.num_queries, embed_dim))
if kv_dim is not None and kv_dim != embed_dim:
self.kv_proj = ReplicatedLinear(
kv_dim,
embed_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.kv_proj",
)
else:
# Maintain the same return value with ReplicatedLinear.forward
self.kv_proj = lambda *args, **kwargs: ( # type: ignore # noqa
nn.Identity()(*args, **kwargs),
None,
)
self.attn = nn.MultiheadAttention(embed_dim, num_heads)
self.ln_q = norm_layer(embed_dim)
self.ln_kv = norm_layer(embed_dim)
self.do_post_projection = do_post_projection
if self.do_post_projection:
self.ln_post = norm_layer(embed_dim)
data = (embed_dim**-0.5) * torch.empty(embed_dim, embed_dim)
self.proj = nn.Parameter(data=data)
def _repeat(self, query, N: int):
return query.unsqueeze(1).repeat(1, N, 1)
class Resampler2(BaseResampler):
"""Resampler-perceiver network to be used for a variety of model types,
e.g., Qwen-vl / Minicpmv 2.0. The main difference is the addition of the
do_post_projection arg, which indicates whether or not there should be
a post layer normalization and projector after the attention. This is
present in minicpmv2.0, but not qwen-vl.
"""
def __init__(
self,
grid_size: int,
embed_dim: int,
num_heads: int,
kv_dim: int | None = None,
norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
adaptive: bool = False,
do_post_projection: bool = True,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__(
grid_size**2,
embed_dim,
num_heads,
kv_dim,
norm_layer,
do_post_projection=do_post_projection,
quant_config=quant_config,
prefix=prefix,
)
self.adaptive = adaptive
pos_embed_arr = get_2d_sincos_pos_embed(embed_dim, grid_size, version=(2, 0))
self.pos_embed = nn.Parameter(
torch.from_numpy(pos_embed_arr).requires_grad_(False)
)
def forward(
self,
x: torch.Tensor,
tgt_sizes: torch.Tensor | None = None,
attn_mask: torch.Tensor | None = None,
) -> torch.Tensor:
if tgt_sizes is None:
tgt_sizes = int(math.sqrt(x.size(1)))
if self.adaptive:
pos_embed_arr = get_2d_sincos_pos_embed(
self.embed_dim, tgt_sizes, version=(2, 0)
)
pos_embed = torch.from_numpy(pos_embed_arr).to(
device=x.device, dtype=x.dtype
)
else:
pos_embed = get_abs_pos(self.pos_embed, tgt_sizes).to(
device=x.device, dtype=x.dtype
)
x, _ = self.kv_proj(x)
x = self.ln_kv(x).permute(1, 0, 2)
N = x.shape[1]
q = self.ln_q(self.query)
out = self.attn(
self._repeat(q, N) + self.pos_embed.unsqueeze(1),
x + pos_embed.unsqueeze(1),
x,
attn_mask=attn_mask,
)[0]
x = out.permute(1, 0, 2)
if self.do_post_projection:
x = self.ln_post(x)
x = x @ self.proj
return x
@@ -1,93 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from .mrope import MRotaryEmbedding
class Ernie4_5_VLRotaryEmbedding(MRotaryEmbedding):
"""3D rotary positional embedding. 3D is t:time h:height w:width."""
def forward_native( # type: ignore[override]
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
assert positions.ndim == 1 or positions.ndim == 2
assert key is not None
num_tokens = positions.shape[-1]
cos_sin = self.cos_sin_cache[positions]
cos, sin = cos_sin.chunk(2, dim=-1)
if positions.ndim == 2:
assert self.mrope_section
section_h = self.mrope_section[0] # 22
section_w = self.mrope_section[1] # 22
section_t = self.mrope_section[2] # 20
assert section_h == section_w
# Split according to [h w h w h w h w... t t t...]
section_cos_t = cos[..., -section_t:]
section_cos_h = cos[..., : section_h + section_w : 2]
section_cos_w = cos[..., 1 : section_h + section_w : 2]
cos_t, cos_h, cos_w = section_cos_t[0], section_cos_h[1], section_cos_w[2]
cos_hw = torch.stack([cos_h, cos_w], dim=-1).reshape(
cos_h.shape[:-1] + (cos_h.shape[-1] * 2,)
)
cos = torch.cat([cos_hw, cos_t], dim=-1)
section_sin_t = sin[..., -section_t:]
section_sin_h = sin[..., : section_h + section_w : 2]
section_sin_w = sin[..., 1 : section_h + section_w : 2]
sin_t, sin_h, sin_w = section_sin_t[0], section_sin_h[1], section_sin_w[2]
sin_hw = torch.stack([sin_h, sin_w], dim=-1).reshape(
sin_h.shape[:-1] + (sin_h.shape[-1] * 2,)
)
sin = torch.cat([sin_hw, sin_t], dim=-1)
query_shape = query.shape
query = query.view(num_tokens, -1, self.head_size)
query_rot = query[..., : self.rotary_dim]
query_pass = query[..., self.rotary_dim :]
query_rot = self.apply_rotary_emb.forward_native(
query_rot,
cos,
sin,
)
query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
key_shape = key.shape
key = key.view(num_tokens, -1, self.head_size)
key_rot = key[..., : self.rotary_dim]
key_pass = key[..., self.rotary_dim :]
key_rot = self.apply_rotary_emb.forward_native(
key_rot,
cos,
sin,
)
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
return query, key
def forward_cuda( # type: ignore[override]
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
return self.forward_native(positions, query, key)
def forward_xpu( # type: ignore[override]
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
# No fused XPU kernel for this 3D t/h/w rope; base
# MRotaryEmbedding.forward_xpu forwards an extra `offsets` arg that
# this class's forward_cuda override doesn't accept. Use native path.
return self.forward_native(positions, query, key)
@@ -1,153 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import async_tensor_h2d
# num_tokens/num_grids vary per request (incl. the ==1 class) and grids_ptr
# is an offset slice of the packed H2D buffer, so its 16B alignment varies
# with grid count; without these the kernel recompiles mid-serving.
@triton.jit(
do_not_specialize=["num_tokens", "num_grids"],
do_not_specialize_on_alignment=["grids_ptr"],
)
def _vit_mrope_setup_kernel(
inv_freq_t_ptr,
inv_freq_h_ptr,
inv_freq_w_ptr,
cos_out_ptr,
sin_out_ptr,
cu_seqlens_ptr, # (G + 1,) int32 token boundaries per grid
grids_ptr, # (G, 3) int32: t, h, w per grid
num_tokens,
num_grids,
half_t: tl.constexpr,
half_h: tl.constexpr,
half_w: tl.constexpr,
spatial_merge: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_D: tl.constexpr,
):
"""Build triton_mrope's (3, N, half_rot) cos/sin t/h/w planes in one
launch: compute each token's (t, h, w) position arithmetically and the
rotation in fp32 from the per-axis inv_freq. Only each axis's mrope
section columns are written; the rest is left uninitialized (the
consumer's per-axis masked loads read only the section columns)."""
pid = tl.program_id(0)
tok = pid * BLOCK_N + tl.arange(0, BLOCK_N)
tok_mask = tok < num_tokens
# Grid (segment) index per token; cu_seqlens is nondecreasing.
seg = tl.zeros([BLOCK_N], dtype=tl.int32)
for g in range(num_grids):
end = tl.load(cu_seqlens_ptr + g + 1)
seg += (tok >= end).to(tl.int32)
seg_start = tl.load(cu_seqlens_ptr + seg, mask=tok_mask, other=0)
grid_h = tl.load(grids_ptr + seg * 3 + 1, mask=tok_mask, other=1)
grid_w = tl.load(grids_ptr + seg * 3 + 2, mask=tok_mask, other=1)
local = tok - seg_start
rem = local % (grid_h * grid_w)
tpos = local // (grid_h * grid_w)
# Within a frame tokens are ordered by spatial-merge block (bh, bw, mh,
# mw), matching the reference ViT position-id permute.
bh = rem // (spatial_merge * grid_w)
r = rem % (spatial_merge * grid_w)
bw = r // (spatial_merge * spatial_merge)
r2 = r % (spatial_merge * spatial_merge)
hpos = bh * spatial_merge + r2 // spatial_merge
wpos = bw * spatial_merge + r2 % spatial_merge
d = tl.arange(0, BLOCK_D)
half_rot = half_t + half_h + half_w
row = (tok * half_rot)[:, None] + d[None, :]
plane = num_tokens * half_rot
# Per-axis compute+store to cap live registers; the fp32 trig sequence
# wants few elements per thread (hence small BLOCK_N, many warps).
in_t = d < half_t
inv_t = tl.load(inv_freq_t_ptr + d, mask=in_t, other=0.0)
freq = tpos.to(tl.float32)[:, None] * inv_t[None, :]
mask = tok_mask[:, None] & in_t[None, :]
tl.store(cos_out_ptr + row, tl.cos(freq), mask=mask)
tl.store(sin_out_ptr + row, tl.sin(freq), mask=mask)
in_h = (d >= half_t) & (d < half_t + half_h)
inv_h = tl.load(inv_freq_h_ptr + d - half_t, mask=in_h, other=0.0)
freq = hpos.to(tl.float32)[:, None] * inv_h[None, :]
mask = tok_mask[:, None] & in_h[None, :]
tl.store(cos_out_ptr + plane + row, tl.cos(freq), mask=mask)
tl.store(sin_out_ptr + plane + row, tl.sin(freq), mask=mask)
in_w = (d >= half_t + half_h) & (d < half_rot)
inv_w = tl.load(inv_freq_w_ptr + d - half_t - half_h, mask=in_w, other=0.0)
freq = wpos.to(tl.float32)[:, None] * inv_w[None, :]
mask = tok_mask[:, None] & in_w[None, :]
tl.store(cos_out_ptr + 2 * plane + row, tl.cos(freq), mask=mask)
tl.store(sin_out_ptr + 2 * plane + row, tl.sin(freq), mask=mask)
def vit_mrope_setup(
inv_freq_t: torch.Tensor,
inv_freq_h: torch.Tensor,
inv_freq_w: torch.Tensor,
grid_thw: list[list[int]],
spatial_merge_size: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""(3, total_N, half_rot_dim) fp32 cos/sin t/h/w planes for triton_mrope.
One fused kernel: positions are computed arithmetically from the grids
and the rotation comes from the per-axis inv_freq, so there is no
cos/sin cache to size or grow.
Args:
inv_freq_t: t-axis inverse frequencies, (t_dim // 2,).
inv_freq_h: h-axis inverse frequencies, (h_dim // 2,).
inv_freq_w: w-axis inverse frequencies, (w_dim // 2,).
grid_thw: per-grid [t, h, w] patch counts (host metadata).
spatial_merge_size: spatial merge factor of the ViT.
"""
device = inv_freq_t.device
cu = [0]
for t, h, w in grid_thw:
cu.append(cu[-1] + t * h * w)
n = cu[-1]
g = len(grid_thw)
# grid_thw is per-request host metadata, so one H2D is unavoidable;
# pack cu_seqlens and grids into a single pinned async copy.
buf = async_tensor_h2d(
cu + [d for grid in grid_thw for d in grid], device=device, dtype=torch.int32
)
cu_seqlens = buf[: g + 1]
grids = buf[g + 1 :].view(g, 3)
half_t = inv_freq_t.numel()
half_h = inv_freq_h.numel()
half_w = inv_freq_w.numel()
half_rot = half_t + half_h + half_w
cos = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
sin = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
_vit_mrope_setup_kernel[(triton.cdiv(n, 32),)](
inv_freq_t,
inv_freq_h,
inv_freq_w,
cos,
sin,
cu_seqlens,
grids,
n,
g,
half_t=half_t,
half_h=half_h,
half_w=half_w,
spatial_merge=spatial_merge_size,
BLOCK_N=32,
BLOCK_D=triton.next_power_of_2(half_rot),
num_warps=8,
)
return cos, sin
@@ -1,83 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""In-place interleaved RoPE on the Q/K slices of a packed QKV buffer."""
import torch
from vllm.triton_utils import tl, triton
# seqlen is not specialized: image sizes vary per request.
@triton.jit(do_not_specialize=["seqlen"])
def _packed_qk_rope_kernel(
qk_ptr, # (seqlen, 2 * nheads, headdim) strided view of the packed QKV
freqs_ptr, # (seqlen, rotary_dim) fp32 re/im view of the complex freqs
seqlen,
qk_row_stride,
qk_head_stride,
rotary_dim: tl.constexpr,
BLOCK_M: tl.constexpr,
):
BLOCK_K: tl.constexpr = triton.next_power_of_2(rotary_dim)
pid_m = tl.program_id(0)
pid_h = tl.program_id(1)
rm = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
rk = tl.arange(0, BLOCK_K)
mask_m = rm < seqlen
base = qk_ptr + pid_h * qk_head_stride + rm[:, None] * qk_row_stride
mask_x = mask_m[:, None] & (rk < rotary_dim)[None, :]
x = tl.load(base + rk[None, :], mask=mask_x, other=0.0).to(tl.float32)
f = tl.load(freqs_ptr + rm[:, None] * rotary_dim + rk[None, :], mask=mask_x)
cos, sin = tl.split(tl.reshape(f, (BLOCK_M, BLOCK_K // 2, 2)))
x0, x1 = tl.split(tl.reshape(x, (BLOCK_M, BLOCK_K // 2, 2)))
o0 = x0 * cos - x1 * sin
o1 = x1 * cos + x0 * sin
tl.store(base + rk[None, :], tl.interleave(o0, o1), mask=mask_x)
def packed_qk_rope_(xqkv: torch.Tensor, freqs_cis: torch.Tensor) -> None:
"""Rotate Q and K in place inside a packed QKV buffer.
Args:
xqkv: contiguous (seqlen, 3, nheads, headdim); only the Q and K
slices are rotated.
freqs_cis: contiguous (seqlen, headdim // 2) complex64 rotary freqs,
read through its interleaved re/im fp32 view.
Matches ``ApplyRotaryEmb(enable_fp32_compute=True)`` applied to Q and K
separately to within one ULP, but in one kernel with ~5x less memory
traffic. Both compute in fp32; the gap is which product the compiler
keeps exact inside the fused multiply-add, which differs per backend.
Requires triton: callers must check HAS_TRITON and use the unfused path
otherwise. Contract violations raise.
"""
assert xqkv.ndim == 4 and xqkv.size(1) == 3, xqkv.shape
assert xqkv.is_contiguous(), xqkv.shape
assert freqs_cis.ndim == 2 and freqs_cis.is_contiguous(), freqs_cis.shape
assert freqs_cis.dtype == torch.complex64, freqs_cis.dtype
seq_length, _, num_heads, head_dim = xqkv.shape
rotary_dim = freqs_cis.size(-1) * 2
assert freqs_cis.size(0) == seq_length, (freqs_cis.shape, xqkv.shape)
assert rotary_dim == head_dim, (freqs_cis.shape, xqkv.shape) # full rotation
qk = xqkv.as_strided(
(seq_length, 2 * num_heads, head_dim),
(3 * num_heads * head_dim, head_dim, 1),
)
BLOCK_M = 4
# seqlen on grid.x: HIP caps gridDim.y at 65535.
grid = (triton.cdiv(seq_length, BLOCK_M), 2 * num_heads)
_packed_qk_rope_kernel[grid](
qk,
freqs_cis.view(torch.float32),
seq_length,
qk.stride(0),
qk.stride(1),
rotary_dim=rotary_dim,
BLOCK_M=BLOCK_M,
num_warps=2 if rotary_dim <= 64 else 4,
)