[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,602 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import cutlass
import cutlass.cute as cute
import torch
from cuda.bindings.driver import CUstream
from cutlass import Float32, Int32, Uint32, Uint64
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import recast_val
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
CuTeDSLLaunchSpec,
VllmCuTeDSLJitKernel,
)
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
)
from vllm.triton_utils import tl, triton
def _dcp_merge_topk_and_interleave(vllm_config: Any) -> tuple[int, int]:
"""(topk, cp_interleave) the DCP merge runs with.
Kpool indexers (GLM-5.x) merge pool ids: topk and the interleave are both
in units of ``index_kpool`` tokens.
"""
hf_text_config = vllm_config.model_config.hf_text_config
topk = hf_text_config.index_topk
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
kpool = getattr(hf_text_config, "index_kpool", None) or 1
if kpool > 1:
topk //= kpool
cp_interleave //= kpool
return topk, cp_interleave
def stable_topk_from_gathered_candidates_cutedsl(
gathered: torch.Tensor,
topk: int,
out: torch.Tensor | None = None,
) -> torch.Tensor:
if out is None:
out = torch.empty(
(gathered.shape[0], topk),
dtype=torch.int32,
device=gathered.device,
)
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL(gathered, out, topk=topk)
return out
def pack_dcp_topk_candidates_cutedsl(
logits: torch.Tensor,
topk_indices: torch.Tensor,
packed: torch.Tensor,
dcp_rank: int,
dcp_world_size: int,
cp_interleave: int,
row_starts: torch.Tensor | None,
) -> None:
topk = topk_indices.shape[1]
row_starts_arg = row_starts if row_starts is not None else topk_indices
_PACK_DCP_TOPK_CANDIDATES_KERNEL(
logits,
topk_indices,
packed,
row_starts_arg,
logits_stride0=logits.stride(0),
logits_stride1=logits.stride(1),
topk_stride0=topk_indices.stride(0),
topk_stride1=topk_indices.stride(1),
packed_stride0=packed.stride(0),
packed_stride1=packed.stride(1),
packed_stride2=packed.stride(2),
num_cols=logits.shape[1],
dcp_rank=dcp_rank,
dcp_world_size=dcp_world_size,
cp_interleave=cp_interleave,
has_row_starts=row_starts is not None,
topk=topk,
block_size=512,
)
class PackDCPTopkCandidatesKernel(
VllmTritonJitKernel["PackDCPTopkCandidatesKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
dcp_rank: int
dcp_world_size: int
cp_interleave: int
has_row_starts: bool
topk: int
block_size: int
# These scalars only describe runtime layouts and bounds; the constexpr
# fields below own the launch geometry and algorithmic specialization.
@staticmethod
@triton.jit(
do_not_specialize=[
"logits_stride0",
"logits_stride1",
"topk_stride0",
"topk_stride1",
"packed_stride0",
"packed_stride1",
"packed_stride2",
"num_cols",
]
)
def kernel(
logits,
topk_indices,
packed,
row_starts,
logits_stride0,
logits_stride1,
topk_stride0,
topk_stride1,
packed_stride0,
packed_stride1,
packed_stride2,
num_cols,
DCP_RANK: tl.constexpr,
DCP_WORLD_SIZE: tl.constexpr,
CP_INTERLEAVE: tl.constexpr,
HAS_ROW_STARTS: tl.constexpr,
TOPK: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0)
tile = tl.program_id(1)
cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = cols < TOPK
local_idx = tl.load(
topk_indices + row * topk_stride0 + cols * topk_stride1,
mask=mask,
other=-1,
)
valid = local_idx >= 0
safe_local_idx = tl.maximum(local_idx, 0)
row_start = 0
if HAS_ROW_STARTS:
row_start = tl.load(row_starts + row)
score_col = safe_local_idx + row_start
score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0))
score = tl.load(
logits + row * logits_stride0 + score_col * logits_stride1,
mask=mask & valid,
other=-float("inf"),
)
global_id = (
(safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE)
+ DCP_RANK * CP_INTERLEAVE
+ safe_local_idx % CP_INTERLEAVE
)
global_id = tl.where(valid, global_id, -1)
packed_base = packed + row * packed_stride0 + cols * packed_stride1
tl.store(packed_base, score, mask=mask)
tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask)
def dispatch( # type: ignore[override]
self,
*,
has_row_starts: bool,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(
**compile_key_fields,
has_row_starts=has_row_starts,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
if dcp_world_size <= 1:
return []
topk, cp_interleave = _dcp_merge_topk_and_interleave(vllm_config)
if topk <= 0:
return []
try:
from vllm.distributed.parallel_state import get_dcp_group
dcp_rank = get_dcp_group().rank_in_group
except Exception:
dcp_rank = 0
return self._trace_dispatch(self.dispatch)(
dcp_rank=dcp_rank,
dcp_world_size=dcp_world_size,
cp_interleave=cp_interleave,
has_row_starts=(False, True),
topk=topk,
block_size=512,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
fp32_ptr = TritonWarmupTensor(torch.float32)
int32_ptr = TritonWarmupTensor(torch.int32)
return dict(
logits=fp32_ptr,
topk_indices=TritonWarmupTensor(torch.int32, shape=(1, compile_key.topk)),
packed=fp32_ptr,
row_starts_arg=int32_ptr,
logits_stride0=1,
logits_stride1=1,
topk_stride0=1,
topk_stride1=1,
packed_stride0=1,
packed_stride1=1,
packed_stride2=1,
num_cols=1,
dcp_rank=compile_key.dcp_rank,
dcp_world_size=compile_key.dcp_world_size,
cp_interleave=compile_key.cp_interleave,
has_row_starts=compile_key.has_row_starts,
topk=compile_key.topk,
block_size=compile_key.block_size,
)
@kernel_launcher
def __call__(
self,
logits: torch.Tensor,
topk_indices: torch.Tensor,
packed: torch.Tensor,
row_starts_arg: torch.Tensor,
*,
logits_stride0: int,
logits_stride1: int,
topk_stride0: int,
topk_stride1: int,
packed_stride0: int,
packed_stride1: int,
packed_stride2: int,
num_cols: int,
dcp_rank: int,
dcp_world_size: int,
cp_interleave: int,
has_row_starts: bool,
topk: int,
block_size: int,
) -> LaunchSpec:
grid = (topk_indices.shape[0], triton.cdiv(topk, block_size))
return grid, dict(
row_starts=row_starts_arg,
DCP_RANK=dcp_rank,
DCP_WORLD_SIZE=dcp_world_size,
CP_INTERLEAVE=cp_interleave,
HAS_ROW_STARTS=has_row_starts,
TOPK=topk,
BLOCK_SIZE=block_size,
num_warps=8,
)
class StableTopKFromGatheredCandidatesKernel(
VllmCuTeDSLJitKernel["StableTopKFromGatheredCandidatesKernel.CompileKey"]
):
tb_size = 512
hist_bins = 2048
radix_bits = (hist_bins - 1).bit_length()
key_bits = Uint64.width
radix_passes = (key_bits + radix_bits - 1) // radix_bits
final_radix_bits = key_bits - radix_bits * (radix_passes - 1)
hist_chunks = (hist_bins + tb_size - 1) // tb_size
warps_per_block = tb_size // cute.arch.WARP_SIZE
@dataclass(frozen=True)
class CompileKey:
topk: int
num_candidates: int
@staticmethod
def kernel(compile_key: CompileKey) -> Any:
tb_size = StableTopKFromGatheredCandidatesKernel.tb_size
hist_bins = StableTopKFromGatheredCandidatesKernel.hist_bins
radix_bits = StableTopKFromGatheredCandidatesKernel.radix_bits
key_bits = StableTopKFromGatheredCandidatesKernel.key_bits
radix_passes = StableTopKFromGatheredCandidatesKernel.radix_passes
final_radix_bits = StableTopKFromGatheredCandidatesKernel.final_radix_bits
hist_chunks = StableTopKFromGatheredCandidatesKernel.hist_chunks
warps_per_block = StableTopKFromGatheredCandidatesKernel.warps_per_block
topk = compile_key.topk
assert hist_bins == 1 << radix_bits
assert compile_key.num_candidates % tb_size == 0, (
"StableTopKFromGatheredCandidatesKernel requires candidate count "
f"to be a multiple of {tb_size}, got {compile_key.num_candidates}"
)
keys_per_thread = compile_key.num_candidates // tb_size
@cute.struct
class SharedStorage:
hist: cute.struct.MemRange[Int32, hist_bins]
committed_count: cute.struct.MemRange[Int32, 1]
running_count: cute.struct.MemRange[Int32, 1]
threshold_bin: cute.struct.MemRange[Int32, 1]
threshold_found: cute.struct.MemRange[Int32, 1]
include_threshold_bin: cute.struct.MemRange[Int32, 1]
prefix_s: cute.struct.Align[cute.struct.MemRange[Uint64, 1], 8]
warp_totals: cute.struct.MemRange[Int32, warps_per_block]
shared_storage = SharedStorage
@cute.jit
def warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32:
for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1):
offset = 1 << i
partial = cute.arch.shuffle_sync_up(
val, offset=offset, mask_and_clamp=0
)
if lane >= offset:
val += partial
return val
@cute.jit
def block_scan_inclusive_i32(
val: Int32,
lane: Int32,
warp_id: Int32,
warp_scratch: cute.Tensor,
warps_per_block: int,
) -> Int32:
prefix = warp_scan_inclusive_i32(val, lane)
if lane == Int32(cute.arch.WARP_SIZE - 1):
warp_scratch[0, warp_id] = prefix
cute.arch.sync_threads()
if warp_id == Int32(0):
warp_total = Int32(0)
if lane < Int32(warps_per_block):
warp_total = warp_scratch[0, lane]
warp_prefix = warp_scan_inclusive_i32(warp_total, lane)
if lane < Int32(warps_per_block):
warp_scratch[0, lane] = warp_prefix - warp_total
cute.arch.sync_threads()
return prefix + warp_scratch[0, warp_id]
@cute.jit
def stable_key(score: Float32, token_id: Int32) -> Uint64:
bits = recast_val(score, Uint32)
mask = Uint32(0x80000000)
if (bits & Uint32(0x80000000)) != Uint32(0):
mask = Uint32(0xFFFFFFFF)
score_key = Uint64(bits ^ mask) << Uint64(32)
id_key = Uint64(~Uint32(token_id))
key = score_key | id_key
if token_id < Int32(0):
key = Uint64(0)
return key
@cute.jit
def prefix_matches(
key: Uint64,
prefix: Uint64,
prefix_bits: Int32,
):
matches = prefix_bits == Int32(0)
if prefix_bits != Int32(0):
shift = Int32(key_bits) - prefix_bits
matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift))
return matches
@cute.jit
def radix_pass(
keys: cute.Tensor,
output: cute.Tensor,
storage,
tid: Int32,
step: Int32,
bits: int,
is_final_pass: bool,
):
hist_smem = storage.hist.get_tensor(cute.make_layout((hist_bins,)))
committed_count_smem = storage.committed_count.data_ptr()
running_count_smem = storage.running_count.data_ptr()
threshold_bin_smem = storage.threshold_bin.data_ptr()
threshold_found_smem = storage.threshold_found.data_ptr()
include_threshold_bin_smem = storage.include_threshold_bin.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
warp_totals_smem = storage.warp_totals.get_tensor(
cute.make_layout((1, warps_per_block))
)
prefix_bits = step * Int32(radix_bits)
num_bins = 1 << bits
block_scan_iterations = (num_bins + tb_size - 1) // tb_size
shift = Int32(key_bits) - prefix_bits - Int32(bits)
bin_mask = Uint64(num_bins - 1)
prefix = prefix_smem.load()
for chunk in cutlass.range_constexpr(hist_chunks):
hist_smem[tid + Int32(chunk * tb_size)] = Int32(0)
if tid == Int32(0):
running_count_smem.store(committed_count_smem.load())
include_threshold_bin_smem.store(Int32(0))
threshold_found_smem.store(Int32(0))
cute.arch.sync_threads()
for key_idx in cutlass.range_constexpr(keys_per_thread):
key = keys[key_idx]
if prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
cute.arch.atomic_add(
hist_smem.iterator + bin_idx,
Int32(1),
sem="relaxed",
scope="cta",
)
cute.arch.sync_threads()
lane = cute.arch.lane_idx()
warp_id = cute.arch.warp_idx()
# Each iteration scans one tb_size-wide slice of bins, high to low.
iter = Int32(0)
threshold_found = threshold_found_smem.load()
while threshold_found == Int32(0) and iter < Int32(block_scan_iterations):
bin_idx = Int32(num_bins - 1) - (iter * Int32(tb_size) + tid)
count = hist_smem[bin_idx]
chunk_inclusive = block_scan_inclusive_i32(
count,
lane,
warp_id,
warp_totals_smem,
warps_per_block,
)
running_count = running_count_smem.load()
prior_in_scan_slice = chunk_inclusive - count
remaining = Int32(topk) - running_count - prior_in_scan_slice
if count > Int32(0) and remaining > Int32(0) and remaining <= count:
threshold_bin_smem.store(bin_idx)
if count <= remaining or cutlass.const_expr(is_final_pass):
include_threshold_bin_smem.store(Int32(1))
threshold_found_smem.store(Int32(1))
# Barrier: every thread must finish reading running_count for this
# slice before tb_size-1 advances it, else a warp racing ahead to
# the store makes a lagging thread double-count the slice total
# (-> remaining too small -> threshold too high -> under-fill).
cute.arch.sync_threads()
if tid == Int32(tb_size - 1):
running_count_smem.store(running_count + chunk_inclusive)
cute.arch.sync_threads()
threshold_found = threshold_found_smem.load()
iter += Int32(1)
threshold = threshold_bin_smem.load()
should_include_threshold = include_threshold_bin_smem.load() != Int32(0)
for key_idx in cutlass.range_constexpr(keys_per_thread):
key = keys[key_idx]
if prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
selected = bin_idx > threshold
if should_include_threshold:
selected = selected or bin_idx == threshold
if selected:
dst = cute.arch.atomic_add(
committed_count_smem,
Int32(1),
sem="relaxed",
scope="cta",
)
if dst < Int32(topk):
output[dst] = recast_val(~Uint32(key), Int32)
cute.arch.sync_threads()
pass_finished = include_threshold_bin_smem.load()
if tid == Int32(0) and pass_finished == Int32(0):
prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift)))
cute.arch.sync_threads()
return pass_finished
@cute.kernel
def device_kernel(input: cute.Tensor, out: cute.Tensor):
row, _, _ = cute.arch.block_idx()
tid, _, _ = cute.arch.thread_idx()
input_row = input[row, None, None]
output_row = out[row, None]
keys = cute.make_rmem_tensor((keys_per_thread,), Uint64)
smem = cutlass.utils.SmemAllocator()
storage = smem.allocate(shared_storage, 8)
committed_count_smem = storage.committed_count.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
for i in range(tid, topk, tb_size):
output_row[i] = Int32(-1)
for key_idx in cutlass.range_constexpr(keys_per_thread):
col = tid + Int32(key_idx * tb_size)
score = Float32(input_row[col, 0])
token_id = Int32(input_row[col, 1])
keys[key_idx] = stable_key(score, token_id)
if tid == Int32(0):
committed_count_smem.store(Int32(0))
prefix_smem.store(Uint64(0))
cute.arch.sync_threads()
step = Int32(0)
finished = Int32(0)
while finished == Int32(0) and step < Int32(radix_passes - 1):
finished = radix_pass(
keys,
output_row,
storage,
tid,
step,
radix_bits,
False,
)
step += Int32(1)
if finished == Int32(0):
radix_pass(
keys,
output_row,
storage,
tid,
Int32(radix_passes - 1),
final_radix_bits,
True,
)
@cute.jit
def host_entrypoint(
gathered: cute.Tensor,
out: cute.Tensor,
stream: CUstream,
):
grid = (gathered.shape[0], 1, 1)
device_kernel(gathered, out).launch(
grid=grid,
block=(tb_size, 1, 1),
stream=stream,
)
return host_entrypoint
def dispatch( # type: ignore[override]
self,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(**compile_key_fields)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
if dcp_world_size <= 1:
return []
topk, _ = _dcp_merge_topk_and_interleave(vllm_config)
if topk <= 0:
return []
return self._trace_dispatch(self.dispatch)(
topk=topk,
num_candidates=topk * dcp_world_size,
)
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
num_rows = cute.sym_int()
gathered = cute.runtime.make_fake_tensor(
Float32,
(num_rows, compile_key.num_candidates, 2),
stride=(cute.sym_int64(divisibility=2), 2, 1),
assumed_align=8,
)
out = make_fake_tensor(
Int32,
(num_rows, compile_key.topk),
divisibility=1,
)
return gathered, out
@kernel_launcher
def __call__(
self,
gathered: torch.Tensor,
out: torch.Tensor,
*,
topk: int,
) -> CuTeDSLLaunchSpec[CompileKey]:
compile_key = self.dispatch(
topk=topk,
num_candidates=gathered.shape[1],
)
return compile_key, (gathered, out)
_PACK_DCP_TOPK_CANDIDATES_KERNEL = PackDCPTopkCandidatesKernel()
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL = StableTopKFromGatheredCandidatesKernel()
@@ -59,10 +59,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
TritonW4A16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
XPUW4A8IntLinearKernel,
XPUwNa16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
ZentorchWNA16LinearKernel,
)
@@ -91,9 +87,6 @@ from vllm.model_executor.kernels.linear.mxfp4.marlin import (
from vllm.model_executor.kernels.linear.mxfp4.pytorch import (
TorchMxFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp4.xpu import (
XPUMxFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp6 import (
MxFp6LinearKernel,
MxFp6LinearLayerConfig,
@@ -132,9 +125,6 @@ from vllm.model_executor.kernels.linear.mxfp8.pytorch import (
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.xpu import (
XPUMxFp8LinearKernel,
)
from vllm.model_executor.kernels.linear.nvfp4 import (
NvFp4LinearKernel,
NvFp4LinearLayerConfig,
@@ -228,11 +218,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import (
TritonFP8ScaledMMLinearKernel,
TritonInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
XPUFp8BlockScaledMMKernel,
XPUW8A8FP8LinearKernel,
XPUW8A16FP8LinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
ZentorchInt8ScaledMMLinearKernel,
)
@@ -373,13 +358,6 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
EmulationMxfp6LinearKernel,
EmulationMxfp4LinearKernel,
},
"xpu": {
XPUW8A8FP8LinearKernel,
XPUFp8BlockScaledMMKernel,
},
"xpu_woq": {
XPUW8A16FP8LinearKernel,
},
}
@@ -444,7 +422,6 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]
HummingInt8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [AiterInt8ScaledMMLinearKernel, TritonInt8ScaledMMLinearKernel],
PlatformEnum.XPU: [TritonInt8ScaledMMLinearKernel],
}
# in priority/performance order (when available)
@@ -473,12 +450,6 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
PerTensorTorchFP8ScaledMMLinearKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
],
PlatformEnum.XPU: [
XPUW8A16FP8LinearKernel,
XPUW8A8FP8LinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
],
}
@@ -504,11 +475,6 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
CPUFp8W8A8BlockScaledMMKernel, # W8A8 preferred; falls back to W8A16 below
CPUFp8BlockScaledMMKernel,
],
PlatformEnum.XPU: [
XPUFp8BlockScaledMMKernel,
TritonFp8BlockScaledMMKernel,
BlockWiseTorchFP8ScaledMMLinearKernel,
],
}
_POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
@@ -522,9 +488,6 @@ _POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]
PlatformEnum.CPU: [
# To be added
],
PlatformEnum.XPU: [
XPUW8A16FP8LinearKernel,
],
}
# in priority/performance order (when available)
@@ -545,10 +508,6 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
ConchLinearKernel,
ExllamaLinearKernel,
],
PlatformEnum.XPU: [
XPUW4A8IntLinearKernel,
XPUwNa16LinearKernel,
],
PlatformEnum.CPU: [
Dynamic4bitLinearKernel,
ZentorchWNA16LinearKernel,
@@ -574,11 +533,6 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
RocmDotScaledMxfp8LinearKernel,
EmulationMxfp8LinearKernel,
],
PlatformEnum.XPU: [
XPUMxFp8LinearKernel,
TorchMxFp8LinearKernel,
EmulationMxfp8LinearKernel,
],
}
_POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
@@ -626,10 +580,6 @@ _POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = {
AiterMxfp4LinearKernel,
EmulationMxfp4LinearKernel,
],
PlatformEnum.XPU: [
XPUMxFp4LinearKernel,
TorchMxFp4LinearKernel,
],
}
# TODO make all kernels inherit from MMLinearKernel
@@ -1348,7 +1298,6 @@ __all__ = [
"MacheteLinearKernel",
"MarlinLinearKernel",
"TritonW4A16LinearKernel",
"XPUW4A8IntLinearKernel",
"XPUwNa16LinearKernel",
"init_mxfp8_linear_kernel",
"Mxfp8LinearKernel",
@@ -1373,7 +1322,6 @@ __all__ = [
"FlashInferCutlassMxfp8LinearKernel",
"FlashInferTrtllmMxfp8LinearKernel",
"MarlinMxfp8LinearKernel",
"XPUMxFp8LinearKernel",
"TorchMxFp8LinearKernel",
"EmulationMxfp8LinearKernel",
"CutlassNvFp4LinearKernel",
@@ -38,10 +38,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
TritonW4A16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
XPUW4A8IntLinearKernel,
XPUwNa16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
ZentorchWNA16LinearKernel,
)
@@ -60,7 +56,6 @@ __all__ = [
"RDNA3W4A16LinearKernel",
"RDNAHybridW4A16LinearKernel",
"TritonW4A16LinearKernel",
"XPUW4A8IntLinearKernel",
"XPUwNa16LinearKernel",
"ZentorchWNA16LinearKernel",
]
@@ -1,218 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch.nn.parameter import Parameter
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
_XPUWNA16_SUPPORTED_QUANT_TYPES = (scalar_types.uint4, scalar_types.uint4b8)
logger = init_logger(__name__)
class XPUwNa16LinearKernel(MPLinearKernel):
@classmethod
def get_min_capability(cls) -> int:
return -1
@classmethod
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUwNa16 only supported on XPU"
if c.act_type != torch.bfloat16 and c.act_type != torch.float16:
return False, "XPUwNa16 only supports BF16/FP16 activations"
if c.weight_type not in _XPUWNA16_SUPPORTED_QUANT_TYPES:
return (
False,
f"Quant type ({c.weight_type}) not supported by "
"XPUwNa16, supported types are: "
f"{_XPUWNA16_SUPPORTED_QUANT_TYPES}",
)
if c.group_size != -1 and c.group_size % 32 != 0:
return (
False,
f"Group size ({c.group_size}) not supported by "
"XPUwNa16, supported group sizes are multiples of 32",
)
if c.partition_weight_shape[0] % 32 != 0:
return (
False,
f"Input size ({c.partition_weight_shape[0]}) not supported by "
"XPUwNa16, supported sizes are multiples of 32",
)
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module):
# Default name since marlin requires empty parameter for zp,
# TODO: remove this requirement from marlin (allow optional tensors)
if self.w_zp_name is None:
self.w_zp_name = "w_zp"
need_transpose = False
qweight_shape = getattr(layer, self.w_q_name).shape
scale_shape = getattr(layer, self.w_s_name).shape
# gptq marlin and compressed tensors wna16 expect different default
# layouts for weight and scale, so we check the shapes to determine
# if we need to transpose
if qweight_shape[0] != scale_shape[0]:
need_transpose = True
if need_transpose:
getattr(layer, self.w_q_name).data = (
getattr(layer, self.w_q_name).data.t().contiguous()
)
getattr(layer, self.w_s_name).data = getattr(layer, self.w_s_name).data
else:
getattr(layer, self.w_s_name).data = (
getattr(layer, self.w_s_name).data.t().contiguous()
)
if self.config.zero_points:
# (FIXME): maybe zero points should also be transposed.
getattr(layer, self.w_zp_name).data = (
getattr(layer, self.w_zp_name).data.t().contiguous()
)
else:
weight_zero_point = torch.Tensor([8]).to(torch.int8).to("xpu")
setattr(
layer, self.w_zp_name, Parameter(weight_zero_point, requires_grad=False)
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
reshaped_x = x.reshape(-1, x.shape[-1])
w_q, w_s, w_zp = self._get_weight_params(layer)
out = torch.ops._xpu_C.int4_gemm_w4a16(
reshaped_x,
w_q.t(),
bias if bias is not None else None,
w_s,
w_zp,
self.config.group_size,
None, # Retained by the external XPU op ABI.
)
return out
class XPUW4A8IntLinearKernel(MPLinearKernel):
"""XPU kernel for W4A8 integer quantization using oneDNN int4_gemm_w4a8.
Weights are symmetric group-quantized int4 packed as uint4.
Activations are dynamically quantized per-token to symmetric int8.
"""
@classmethod
def get_min_capability(cls) -> int:
return -1
@classmethod
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUW4A8Int only supported on XPU"
if c.act_type not in (torch.bfloat16, torch.float16):
return False, "XPUW4A8Int requires BF16/FP16 activations"
if c.weight_type != scalar_types.int4:
return (
False,
f"XPUW4A8Int requires int4 weights, got {c.weight_type}",
)
if c.zero_points:
return False, "XPUW4A8Int only supports symmetric weight quantization"
if c.group_size != -1 and c.group_size % 32 != 0:
return (
False,
f"Group size ({c.group_size}) not supported by XPUW4A8Int, "
"must be a multiple of 32",
)
in_size, out_size = c.partition_weight_shape
if in_size % 8 != 0 or out_size % 8 != 0:
return (
False,
f"in/out sizes ({in_size}, {out_size}) must be multiples of 8",
)
if c.act_type != torch.float16:
logger.warning_once(
"XPUW4A8IntLinearKernel is running with model dtype %s, "
"but int4_gemm_w4a8 produces float16 output. Recommend "
"setting --dtype float16 for best performance.",
c.act_type,
)
return True, None
def _pack_int4_weight(self, w: torch.Tensor) -> torch.Tensor:
# w is [N, K] int8 with values in [-8, 7]
w_u4 = w.to(torch.int32) + 8 # shift to [0, 15]
w_u4 = w_u4.reshape(w.shape[0], w.shape[1] // 8, 8) # [N, K/8, 8]
shifts = torch.arange(0, 32, 4, dtype=torch.int32, device=w.device)
packed = ((w_u4 & 0xF) << shifts[None, None, :]).sum(dim=2).to(torch.int32)
return packed
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight_scale.data = layer.weight_scale.data.t().contiguous()
device = layer.weight_packed.device
# TODO: support asymmetric quantization
weight_zero_point = torch.tensor([8], dtype=torch.int8, device=device)
layer.weight_zero_point = Parameter(weight_zero_point, requires_grad=False)
# weight_packed is [out, in] int8, signed int4 values in [-8, 7]
w = layer.weight_packed.data # [out, in]
# TODO: implement asym case
packed = self._pack_int4_weight(w) # [out, in/8] packed uint4
replace_parameter(
layer,
self.w_q_name,
torch.nn.Parameter(packed, requires_grad=False),
)
# Free the original unpacked int8 weight (still registered as "weight")
# to avoid double-storing both int8 [N, K] and int32 [N, K/8] in memory.
delattr(layer, "weight")
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
reshaped_x = x.reshape(-1, x.shape[-1]) # [M, K]
from vllm._xpu_ops import xpu_ops as ops
# TODO: static and asymmetric quantization case
# Common code for CompressedTensorsW4A8Int does not read act symmetry data
quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref(
reshaped_x, True, 8
)
out = torch.ops._xpu_C.int4_gemm_w4a8(
quant_x,
x_scale,
x_zero,
layer.weight_packed.t(),
layer.weight_scale,
layer.weight_zero_point,
self.config.group_size,
None, # Retained by the external XPU op ABI.
bias,
)
return out.to(x.dtype)
@@ -1,58 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
xpu_mxfp4_quantize as quant_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
class XPUMxFp4LinearKernel(MxFp4LinearKernel):
"""MXFP4 W4A4 GEMM on XPU."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUMxFp4 only support on XPU"
return True, None
@classmethod
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
if config.activation_quant_key != kMxfp4Dynamic:
return False, "only supports MXFP4 dynamic activation"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
weight = layer.weight.view(torch.float4_e2m1fn_x2)
replace_parameter(layer, "weight", weight.data.t())
weight_scale = layer.weight_scale.view(torch.float8_e8m0fnu)
weight_scale = weight_scale.t().contiguous()
replace_parameter(layer, "weight_scale", weight_scale.data)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
out_dtype = x.dtype
x_fp4, x_blockscale = quant_mxfp4(x)
return torch.ops._xpu_C.fp4_gemm(
x_fp4,
layer.weight,
x_blockscale,
layer.weight_scale,
out_dtype,
bias,
)
@@ -1,82 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
xpu_mxfp8_quantize as quant_mxfp8,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
class XPUMxFp8LinearKernel(Mxfp8LinearKernel):
"""MXFP8 W8A8 GEMM on XPU."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUMxFp8 only support on XPU"
return True, None
@classmethod
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Checkpoint scale is [N, K//32] (one E8M0 scale per 32 elements).
# oneDNN fp8_gemm requires contiguous [K//32, N] layout. Store the
# transposed contiguous buffer as a .t() view so that:
# - dequantization consumers still see the checkpoint shape
# - apply_weights recovers the oneDNN layout via .t() at zero cost
weight_scale = layer.weight_scale.view(torch.float8_e8m0fnu)
scale_kn = weight_scale.data.t().contiguous()
replace_parameter(layer, "weight_scale", scale_kn.t())
if getattr(layer, "is_bmm", False):
self._prepare_bmm_params(layer, scale_kn)
def _prepare_bmm_params(
self, layer: torch.nn.Module, scale_kn: torch.Tensor
) -> None:
"""Precompute batched weight and scale for grouped fp8_bmm (e.g. wo_a).
Splits scale [K//32, N_total] into [G, K//32, N_per_group] and weight
[N_total, K] into contiguous [G, K, N_per_group] for batch GEMM.
"""
batch = layer.bmm_batch_size
k_blocks, n_blocks = scale_kn.shape
layer.bmm_scale = (
scale_kn.reshape(k_blocks, batch, n_blocks // batch)
.permute(1, 0, 2)
.contiguous()
)
w = layer.weight
n_total, k = w.shape
layer.bmm_weight = (
w.reshape(batch, n_total // batch, k).permute(0, 2, 1).contiguous()
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
out_dtype = x.dtype
x_fp8, x_scale = quant_mxfp8(x)
# Weight is [N, K]; .t() gives a [K, N] view without copying.
# Scale is stored as [N, K//32]; .t() recovers the contiguous
# [K//32, N] buffer that oneDNN expects.
return torch.ops._xpu_C.fp8_gemm(
x_fp8,
layer.weight.t(),
out_dtype,
x_scale,
layer.weight_scale.t(),
bias,
)
@@ -42,9 +42,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import (
TritonFP8ScaledMMLinearKernel,
TritonInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
XPUFp8BlockScaledMMKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
ZentorchInt8ScaledMMLinearKernel,
)
@@ -73,5 +70,4 @@ __all__ = [
"CPUFp8BlockScaledMMKernel",
"CPUFp8PerTensorScaledMMLinearKernel",
"CPUFP8W8A8ScaledMMLinearKernel",
"XPUFp8BlockScaledMMKernel",
]
@@ -1,284 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Sequence
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel
from .ScaledMMLinearKernel import FP8ScaledMMLinearKernel, FP8ScaledMMLinearLayerConfig
class XPUW8A8FP8LinearKernel(FP8ScaledMMLinearKernel):
_SUPPORTED_ACT_QUANT_KEYS = {
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
}
_SUPPORTED_WEIGHT_QUANT_KEYS = {
kFp8StaticChannelSym,
kFp8StaticTensorSym,
}
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUW8A8FP8Linear only support on XPU"
return True, None
@classmethod
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
if c.weight_quant_key not in cls._SUPPORTED_WEIGHT_QUANT_KEYS:
return (
False,
"XPUW8A8FP8Linear only support per-channel and per-tensor quantization",
)
if c.activation_quant_key not in cls._SUPPORTED_ACT_QUANT_KEYS:
return (
False,
"XPUW8A8FP8Linear only support per-tensor and per-token activation "
"quantization",
)
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
return False, "XPUW8A8FP8Linear only support FP8 weight dtype"
if c.activation_quant_key.dtype not in {
torch.float8_e5m2,
torch.float8_e4m3fn,
}:
return False, "XPUW8A8FP8Linear only support FP8 activation dtype"
return True, None
def __init__(
self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str]
) -> None:
super().__init__(c, layer_param_names)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Ensure weight is stored as C-contiguous [K, N] (KN layout).
Checkpoints store weight as [N, K]; fp8_gemm requires [K, N],
C-contiguous. Three incoming layouts are possible:
• [N, K] C-contiguous ← direct checkpoint → .t().contiguous()
• [K, N] Fortran-order ← fp8.py's weight.t() → .contiguous()
• [K, N] C-contiguous ← already correct → no-op
For square weights (K == N) the shape is ambiguous; contiguity is used
as a proxy: C-contiguous ≡ checkpoint [N, K] (needs transpose);
Fortran-order ≡ fp8.py already transposed (needs only contiguous).
"""
K = getattr(layer, "input_size_per_partition", self.config.weight_shape[1])
N = getattr(layer, "output_size_per_partition", self.config.weight_shape[0])
w = layer.weight
if w.shape not in {(K, N), (N, K)}:
raise ValueError(
f"XPUFP8ScaledMM expects weight shape ({K},{N}) or ({N},{K}), "
f"but got {tuple(w.shape)}"
)
needs_transpose = w.shape == (N, K) if K != N else w.is_contiguous()
layer_weight = w.t() if needs_transpose else w
replace_parameter(layer, "weight", layer_weight)
ws = layer.weight_scale
if ws.numel() == 1:
replace_parameter(layer, "weight_scale", ws.reshape(1))
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
# B is C-contiguous [K, N] from process_weights_after_loading.
# fp8_gemm routes on scale dtype (float32) and numel:
# As [1] → per-tensor (numel==1 branch)
# As [M,1] → per-token (group={1,K} branch, broadcast across K)
# Bs [1] → per-tensor
# Bs [N] → per-channel (mask=bit1 branch)
# No shape manipulation needed here.
output = torch.ops._xpu_C.fp8_gemm(A, B, out_dtype, As, Bs, bias)
return output.view(*output_shape)
class XPUW8A16FP8LinearKernel(FP8ScaledMMLinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUW8A16FP8Linear only support on XPU"
return True, None
@classmethod
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
if c.weight_quant_key not in {kFp8StaticChannelSym, kFp8StaticTensorSym}:
return (
False,
"XPUW8A16FP8Linear only support per-channel and per-tensor "
"quantization",
)
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
return False, "XPUW8A16FP8Linear only support FP8 weight dtype"
return True, None
def __init__(
self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str]
) -> None:
assert self.can_implement(c)[0]
assert self.is_supported()[0]
self.config = c
self.layer_param_names = layer_param_names
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# fp8_gemm_w8a16 expects weight in [in, out] layout.
# Transpose if weight is still in [out, in] layout.
# For square matrices, use contiguity as tie-breaker:
# checkpoint weights are contiguous, .t() views are not.
weight = layer.weight
out_features, in_features = self.config.weight_shape
if weight.shape == (out_features, in_features) and (
in_features != out_features or weight.is_contiguous()
):
replace_parameter(layer, "weight", weight.data.t())
# else: already in [in, out] layout — no-op
weight_scale = layer.weight_scale.t().contiguous()
replace_parameter(layer, "weight_scale", weight_scale.data)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
weight = layer.weight
weight_scale = layer.weight_scale
return torch.ops._xpu_C.fp8_gemm_w8a16(x, weight, weight_scale, bias)
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
pass
class XPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUFp8BlockScaledMM only support on XPU"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module):
super().process_weights_after_loading(layer)
scale_attr = (
"weight_scale_inv" if hasattr(layer, "weight_scale_inv") else "weight_scale"
)
scale = getattr(layer, scale_attr)
# Ragged N (N % block_n != 0): oneDNN needs n_blocks to divide N.
# Weight untouched; only repeat scale rows to a finer N-group gn that
# divides both N and block_n (gn = gcd(N, block_n)):
# scale [ceil(N/block_n), K/block_k] --> [N/gn, K/block_k]
# oneDNN only accepts gn that is a multiple of 16, and gcd(N, block_n)
# is a power of two (block_n=128), so gn must be >= 16. No-op when
# N % block_n == 0.
block_n, block_k = self.weight_group_shape
N, K = layer.weight.shape
if N % block_n != 0:
gn = math.gcd(N, block_n)
assert gn % 16 == 0, (
f"XPU block-scaled FP8: N ({N}) yields group width {gn}, but "
f"oneDNN only supports multiples of 16; this weight shape is "
f"unsupported."
)
col_start = torch.arange(N // gn, device=scale.device) * gn
src_idx = torch.div(col_start, block_n, rounding_mode="floor")
scale = scale.index_select(0, src_idx).contiguous()
# Ragged K needs the runtime activation scale expanded too, which we
# don't handle; DeepSeek/GLM keep K block-aligned, so fail loudly.
assert K % block_k == 0, (
f"XPU block-scaled FP8 requires K ({K}) to be a multiple of the "
f"weight block size ({block_k}); ragged-K weights are unsupported."
)
# Checkpoint scale is [n_blocks, k_blocks] (one value per block tile).
# oneDNN fp8_gemm requires contiguous [k_blocks, n_blocks] layout.
# We store the transposed contiguous buffer as a .t() view so that:
# - MLA's scaled_dequantize still sees [n_blocks, k_blocks] shape
# - apply_block_scaled_mm recovers the contiguous buffer via .t()
scale_kn = scale.data.t().contiguous() # [k_blocks, n_blocks]
replace_parameter(layer, scale_attr, scale_kn.t()) # view: [n_blocks, k_blocks]
if getattr(layer, "is_bmm", False):
self._prepare_bmm_params(layer, scale_kn)
def _prepare_bmm_params(
self, layer: torch.nn.Module, scale_kn: torch.Tensor
) -> None:
"""Precompute batched weight and scale for grouped fp8_bmm (e.g. wo_a).
Splits scale [k_blocks, n_blocks] into [G, k_blocks, n_blocks_per_group]
and weight [N_total, K] into [G, K, N_per_group] for batch GEMM.
"""
batch = layer.bmm_batch_size
k_blocks, n_blocks = scale_kn.shape
layer.bmm_scale = (
scale_kn.reshape(k_blocks, batch, n_blocks // batch)
.permute(1, 0, 2)
.contiguous()
)
w = layer.weight
N_total, K = w.shape
layer.bmm_weight = w.reshape(batch, N_total // batch, K).permute(
0, 2, 1
) # [G, K, N_per_group]
def apply_block_scaled_mm(
self,
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
Bs: torch.Tensor,
) -> torch.Tensor:
# B is [N, K]; .t() gives [K, N] view (no copy).
# Bs is stored as [n_blocks, k_blocks] view; .t() recovers the
# contiguous [k_blocks, n_blocks] buffer that oneDNN expects.
return torch.ops._xpu_C.fp8_gemm(
A,
B.t(),
self.config.out_dtype,
As,
Bs.t(),
torch.Tensor(),
)
@@ -1,36 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .aiter import *
from .cpu import *
from .tilelang import *
from .torch import *
from .triton import *
__all__ = [
"mhc_pre_cuda",
"mhc_post_cuda",
"mhc_fused_post_pre_cuda",
"hc_head_fused_kernel_cuda",
"mhc_pre_aiter",
"mhc_pre_delayed_aiter",
"mhc_fused_post_pre_delayed_rms_norm_aiter",
"mhc_post_aiter",
"mhc_fused_post_pre_aiter",
"hc_head_fused_aiter",
"mhc_pre_cpu",
"mhc_post_cpu",
"hc_head_fused_cpu",
"mhc_pre_tilelang",
"mhc_post_tilelang",
"mhc_fused_post_pre_tilelang",
"hc_head_fused_tilelang",
"mhc_pre_torch",
"mhc_post_torch",
"mhc_fused_post_pre_torch",
"hc_head_fused_torch",
"mhc_pre_mix_triton",
"mhc_pre_triton",
"mhc_post_triton",
"mhc_fused_post_pre_triton",
"hc_head_fused_triton",
]
-428
View File
@@ -1,428 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.utils.torch_utils import direct_register_custom_op
def mhc_pre_aiter(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
n_splits: split-k factor;
norm_weight: optional RMSNorm weight fused into the pre kernel
norm_eps: epsilon for the fused RMSNorm when norm_weight is set
Returns:
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_pre(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_weight,
norm_eps,
)
def _mhc_pre_aiter_fake(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
# Create empty tensors with correct shapes for meta device / shape inference
post_mix = torch.empty(
*outer_shape,
hc_mult,
1,
dtype=torch.float32,
device=residual.device,
)
comb_mix = torch.empty(
*outer_shape,
hc_mult,
hc_mult,
dtype=torch.float32,
device=residual.device,
)
layer_input = torch.empty(
*outer_shape,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
)
return post_mix, comb_mix, layer_input
def mhc_pre_delayed_aiter(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
residual_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""MHC pre with the pre-mix carried in from the previous sublayer.
Matches ``mhc_pre_delayed_torch``: the stream collapse uses *pre_mix*
rather than the gate computed here, and that gate is returned as the
pre-mix for the next sublayer seam.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
pre_mix: shape (..., hc_mult) from the previous sublayer, or None at
model entry to select residual stream zero.
sublayer_out: attention or FFN output, shape (..., hidden_size). When
given with the two mixes below, the preceding post block is
applied here so AITER can fold it into the pre projection.
post_layer_mix: shape (..., hc_mult, 1), post gate for that block.
comb_res_mix: shape (..., hc_mult, hc_mult), residual comb for it.
residual_out: shape (..., hc_mult, hidden_size), written with the post
block's new residual. Required exactly when the post is folded in.
Returns:
post_mix: shape (..., hc_mult, 1), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
next_pre_mix: shape (..., hc_mult), dtype torch.float32
"""
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_pre_delayed(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
sublayer_out,
post_layer_mix,
comb_res_mix,
residual_out,
)
def _mhc_pre_delayed_aiter_fake(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
residual_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
return (
torch.empty(
*outer_shape, hc_mult, 1, dtype=torch.float32, device=residual.device
),
torch.empty(
*outer_shape, hc_mult, hc_mult, dtype=torch.float32, device=residual.device
),
torch.empty(
*outer_shape, hidden_size, dtype=torch.bfloat16, device=residual.device
),
torch.empty(*outer_shape, hc_mult, dtype=torch.float32, device=residual.device),
)
def mhc_fused_post_pre_delayed_rms_norm_aiter(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None,
sublayer_out: torch.Tensor | None,
post_layer_mix: torch.Tensor | None,
comb_res_mix: torch.Tensor | None,
norm_weight: torch.Tensor,
norm_eps: float,
residual_out: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
""":func:`mhc_pre_delayed_aiter` with the RMSNorm of the collapse folded in:
``layer_input`` comes back normalised with ``norm_weight`` / ``norm_eps``."""
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_fused_post_pre_delayed_rms_norm(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
sublayer_out,
post_layer_mix,
comb_res_mix,
norm_weight,
norm_eps,
residual_out,
)
def _mhc_fused_post_pre_delayed_rms_norm_aiter_fake(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None,
sublayer_out: torch.Tensor | None,
post_layer_mix: torch.Tensor | None,
comb_res_mix: torch.Tensor | None,
norm_weight: torch.Tensor,
norm_eps: float,
residual_out: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
return _mhc_pre_delayed_aiter_fake(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
def mhc_post_aiter(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_post(
x,
residual,
post_layer_mix,
comb_res_mix,
)
def _mhc_post_aiter_fake(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
return torch.empty_like(residual)
def mhc_fused_post_pre_aiter(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
tile_n: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Fused mHC post + next mHC pre on ROCm via AITER.
Returns residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur.
"""
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_fused_post_pre(
x,
residual,
post_layer_mix,
comb_res_mix,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_weight,
norm_eps,
)
def _mhc_fused_post_pre_aiter_fake(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
tile_n: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
post_mix = torch.empty(
*outer_shape,
hc_mult,
1,
dtype=torch.float32,
device=residual.device,
)
comb_mix = torch.empty(
*outer_shape,
hc_mult,
hc_mult,
dtype=torch.float32,
device=residual.device,
)
layer_input = torch.empty(
*outer_shape,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
)
return torch.empty_like(residual), post_mix, comb_mix, layer_input
direct_register_custom_op(
op_name="mhc_pre_aiter",
op_func=mhc_pre_aiter,
mutates_args=[],
fake_impl=_mhc_pre_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_pre_delayed_aiter",
op_func=mhc_pre_delayed_aiter,
mutates_args=["residual_out"],
fake_impl=_mhc_pre_delayed_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_fused_post_pre_delayed_rms_norm_aiter",
op_func=mhc_fused_post_pre_delayed_rms_norm_aiter,
mutates_args=["residual_out"],
fake_impl=_mhc_fused_post_pre_delayed_rms_norm_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_post_aiter",
op_func=mhc_post_aiter,
mutates_args=[],
fake_impl=_mhc_post_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_fused_post_pre_aiter",
op_func=mhc_fused_post_pre_aiter,
mutates_args=[],
fake_impl=_mhc_fused_post_pre_aiter_fake,
)
-96
View File
@@ -1,96 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm import _custom_ops as ops
def mhc_pre_cpu(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""CPU-ported mHC pre block (see `mhc_pre_torch` for the eager reference).
The ported kernel (`hc_pre_fused_cpu`) only exposes one merged `hc_eps`
(used for both `hc_pre_eps`/`hc_sinkhorn_eps`) and hardcodes the post-mix
multiplier to 2.0 -- true of every real call site in
`models/deepseek_v4/cpu/model.py`, so this is not a capability loss here.
"""
assert n_splits == 1, "mhc_pre_cpu does not support n_splits != 1"
assert hc_pre_eps == hc_sinkhorn_eps, (
"mhc_pre_cpu requires hc_pre_eps == hc_sinkhorn_eps (single merged hc_eps)"
)
assert hc_post_mult_value == 2.0, "mhc_pre_cpu hardcodes the post-mix multiplier"
hc_mult, hidden_size = residual.shape[-2:]
outer_shape = residual.shape[:-2]
x_flat = residual.reshape(-1, hc_mult, hidden_size)
layer_input, post, comb = ops.hc_pre_fused_cpu(
x_flat,
fn,
hc_scale,
hc_base,
hc_mult,
sinkhorn_repeat,
rms_eps,
hc_pre_eps,
)
post_mix = post.view(*outer_shape, hc_mult, 1)
comb_mix = comb.view(*outer_shape, hc_mult, hc_mult)
layer_input = layer_input.view(*outer_shape, hidden_size)
return post_mix, comb_mix, layer_input
def mhc_post_cpu(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
"""CPU-ported mHC post block (see `mhc_post_torch` for the eager reference)."""
hc_mult, hidden_size = residual.shape[-2:]
outer_shape = residual.shape[:-2]
x_flat = x.reshape(-1, hidden_size)
residual_flat = residual.reshape(-1, hc_mult, hidden_size)
post_flat = post_layer_mix.reshape(-1, hc_mult).float()
comb_flat = comb_res_mix.reshape(-1, hc_mult, hc_mult).float()
out = ops.hc_post_fused_cpu(x_flat, residual_flat, post_flat, comb_flat)
return out.view(*outer_shape, hc_mult, hidden_size)
def hc_head_fused_cpu(
hidden_states: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_norm_eps: float,
hc_eps: float,
) -> torch.Tensor:
"""CPU-ported HC head reduction (see `test_hc_head_cpu` in
tests/kernels/mhc/test_mhc_kernels.py for the eager reference this is tested
against).
The ported kernel's C++ signature takes `(hc_eps, norm_eps)` -- the
opposite order from this wrapper's `(rms_norm_eps, hc_eps)`, which matches
the real call site in `models/deepseek_v4/cpu/model.py`.
"""
hc_mult, hidden_size = hidden_states.shape[-2:]
outer_shape = hidden_states.shape[:-2]
hs_flat = hidden_states.reshape(-1, hc_mult, hidden_size)
out = ops.hc_head_fused_cpu(hs_flat, hc_fn, hc_scale, hc_base, hc_eps, rms_norm_eps)
return out.view(*outer_shape, hidden_size)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-147
View File
@@ -1,147 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
def mhc_pre_delayed_torch(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
x: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Reference for mHC pre using coefficients from the previous sublayer."""
hc_mult = residual.shape[1]
x = (residual.flatten(1) if x is None else x).float()
mixes = (x @ fn.t()) * torch.rsqrt(x.square().mean(-1, keepdim=True) + rms_eps)
pre = (
torch.sigmoid(mixes[:, :hc_mult] * hc_scale[0] + hc_base[:hc_mult]) + hc_pre_eps
)
post = (
torch.sigmoid(
mixes[:, hc_mult : 2 * hc_mult] * hc_scale[1]
+ hc_base[hc_mult : 2 * hc_mult]
)
* hc_post_mult_value
)
comb = mixes[:, 2 * hc_mult :].view(-1, hc_mult, hc_mult) * hc_scale[2]
comb = comb + hc_base[2 * hc_mult :].view(1, hc_mult, hc_mult)
comb = torch.softmax(comb, dim=-1) + hc_sinkhorn_eps
comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
for _ in range(sinkhorn_repeat - 1):
comb = comb / (comb.sum(dim=-1, keepdim=True) + hc_sinkhorn_eps)
comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
layer_input = (
residual[:, 0]
if pre_mix is None
else (pre_mix.unsqueeze(-1) * residual.float()).sum(dim=1).to(residual.dtype)
)
return post.unsqueeze(-1), comb, layer_input, pre
def mhc_pre_torch(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
n_splits: split-k factor;
Returns:
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
# Validate shapes
assert residual.dtype == torch.bfloat16
assert fn.dtype == torch.float32
assert hc_scale.dtype == torch.float32
assert hc_base.dtype == torch.float32
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
hc_mult2 = hc_mult * hc_mult
hc_mult3 = hc_mult * 2 + hc_mult2
hc_hidden_size = hc_mult * hidden_size
assert fn.shape[0] == hc_mult3
assert fn.shape[1] == hc_hidden_size
assert hc_scale.shape == (3,)
assert hc_base.shape == (hc_mult3,)
outer_shape = residual.shape[:-2]
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
fn_flat = fn
x = residual_flat.view(num_tokens, hc_mult * hidden_size).to(torch.float32)
mixes = torch.matmul(x, fn_flat.t())
sqrsum = x.square().sum(dim=-1, keepdim=True)
mixes = mixes * torch.rsqrt(sqrsum / (hc_mult * hidden_size) + rms_eps)
pre_logits = mixes[:, :hc_mult] * hc_scale[0] + hc_base[:hc_mult]
pre_mix = torch.sigmoid(pre_logits) + hc_pre_eps
post_logits = (
mixes[:, hc_mult : 2 * hc_mult] * hc_scale[1] + hc_base[hc_mult : 2 * hc_mult]
)
post_mix = torch.sigmoid(post_logits) * hc_post_mult_value
comb_logits = mixes[:, 2 * hc_mult :].view(num_tokens, hc_mult, hc_mult) * hc_scale[
2
] + hc_base[2 * hc_mult :].view(1, hc_mult, hc_mult)
comb_mix = torch.softmax(comb_logits, dim=-1) + hc_sinkhorn_eps
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
for _ in range(sinkhorn_repeat - 1):
comb_mix = comb_mix / (comb_mix.sum(dim=-1, keepdim=True) + hc_sinkhorn_eps)
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
layer_input = torch.sum(
pre_mix.unsqueeze(-1) * residual_flat.to(torch.float32), dim=1
).to(torch.bfloat16)
return (
post_mix.view(*outer_shape, hc_mult, 1),
comb_mix.view(*outer_shape, hc_mult, hc_mult),
layer_input.view(*outer_shape, hidden_size),
)
def mhc_post_torch(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
mixed_residual = torch.einsum(
"...ij,...ih->...jh",
comb_res_mix.to(torch.float32),
residual.to(torch.float32),
)
post_term = post_layer_mix.to(torch.float32) * x.unsqueeze(-2).to(torch.float32)
return (mixed_residual + post_term).to(residual.dtype)
-349
View File
@@ -1,349 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn.functional as F
from torch import Tensor
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
@triton.jit
def _rmsnorm_nw_kernel(
x_ptr,
out_ptr,
stride_row,
D,
eps,
RBLOCK: tl.constexpr,
):
"""Weight-free RMSNorm Triton kernel: out = x * rsqrt(mean(x², -1) + eps)."""
row = tl.program_id(0)
cols = tl.arange(0, RBLOCK)
mask = cols < D
x = tl.load(
x_ptr + row * stride_row + cols,
mask=mask,
other=0.0,
eviction_policy="evict_first",
).to(tl.float32)
var = tl.sum(x * x, 0) / D
rstd = tl.rsqrt(var + eps)
out = (x * rstd).to(out_ptr.dtype.element_ty)
tl.store(out_ptr + row * D + cols, out, mask=mask, eviction_policy="evict_first")
def rmsnorm_nw(x: Tensor, eps: float) -> Tensor:
"""Weight-free RMSNorm over the last dimension.
Treats *x* as ``[num_rows, D]`` where ``num_rows = product(shape[:-1])``.
Returns a contiguous tensor with the same shape and dtype as *x*.
"""
orig_shape = x.shape
D = orig_shape[-1]
x_2d = x.reshape(-1, D)
num_rows = x_2d.shape[0]
out = torch.empty_like(x_2d)
RBLOCK = triton.next_power_of_2(D)
_rmsnorm_nw_kernel[(num_rows,)](
x_2d,
out,
x_2d.stride(0),
D,
eps,
RBLOCK=RBLOCK,
num_warps=1 if RBLOCK <= 512 else (4 if RBLOCK <= 4096 else 8),
)
return out.view(orig_shape)
@triton.jit
def _hc_head_reduce_store_kernel(
pre_ptr,
x_ptr,
out_ptr,
hidden_size: tl.constexpr,
hc_mult: tl.constexpr,
pre_stride_t: tl.constexpr,
pre_stride_m: tl.constexpr,
x_stride_t: tl.constexpr,
x_stride_m: tl.constexpr,
x_stride_h: tl.constexpr,
out_stride_t: tl.constexpr,
out_stride_h: tl.constexpr,
BLOCK_H: tl.constexpr,
):
token_idx = tl.program_id(0).to(tl.int64)
block_idx = tl.program_id(1)
offsets = block_idx * BLOCK_H + tl.arange(0, BLOCK_H)
mask = offsets < hidden_size
acc = tl.zeros((BLOCK_H,), dtype=tl.float32)
for mix_idx in tl.static_range(0, hc_mult):
pre = tl.load(pre_ptr + token_idx * pre_stride_t + mix_idx * pre_stride_m).to(
tl.float32
)
x = tl.load(
x_ptr
+ token_idx * x_stride_t
+ mix_idx * x_stride_m
+ offsets * x_stride_h,
mask=mask,
other=0.0,
).to(tl.float32)
acc += pre * x
tl.store(
out_ptr + token_idx * out_stride_t + offsets * out_stride_h,
acc,
mask=mask,
)
def hc_collapse_triton(x: Tensor, pre_mix: Tensor) -> Tensor:
"""Collapse BF16 residual streams with FP32 pre-mix coefficients."""
assert x.ndim == 3 and x.dtype == torch.bfloat16
num_tokens, hc_mult, hidden_size = x.shape
assert pre_mix.shape == (num_tokens, hc_mult)
assert pre_mix.dtype == torch.float32
out = torch.empty(num_tokens, hidden_size, dtype=x.dtype, device=x.device)
if num_tokens == 0:
return out
block_h = 1024
_hc_head_reduce_store_kernel[(num_tokens, triton.cdiv(hidden_size, block_h))](
pre_mix,
x,
out,
hidden_size,
hc_mult,
pre_mix.stride(0),
pre_mix.stride(1),
x.stride(0),
x.stride(1),
x.stride(2),
out.stride(0),
out.stride(1),
BLOCK_H=block_h,
num_warps=4,
# Preserve the separate FP32 multiply and sum in the Torch reference.
enable_fp_fusion=False,
)
return out
def _hc_collapse_triton_fake(x: Tensor, pre_mix: Tensor) -> Tensor:
return torch.empty(x.shape[0], x.shape[2], dtype=x.dtype, device=x.device)
@triton.jit
def _mhc_pre_mix_kernel(
gemm_ptr,
sqrsum_ptr,
hc_scale_ptr,
hc_base_ptr,
out_ptr,
splitk,
hc_mult: tl.constexpr,
gemm_stride_k,
gemm_stride_t,
gemm_stride_j,
sqrsum_stride_k,
sqrsum_stride_t,
out_stride_t,
out_stride_j,
inv_hc_hidden,
rms_eps,
hc_pre_eps,
SPLITK_BLOCK: tl.constexpr,
HC_BLOCK: tl.constexpr,
):
"""Recover the pre-mix gate from a split-k mHC pre GEMM output."""
token_idx = tl.program_id(0).to(tl.int64)
ks = tl.arange(0, SPLITK_BLOCK)
js = tl.arange(0, HC_BLOCK)
kmask = ks < splitk
jmask = js < hc_mult
# Only the first hc_mult GEMM columns feed the pre gate.
gemm = tl.load(
gemm_ptr
+ ks[:, None] * gemm_stride_k
+ token_idx * gemm_stride_t
+ js[None, :] * gemm_stride_j,
mask=kmask[:, None] & jmask[None, :],
other=0.0,
).to(tl.float32)
mixes = tl.sum(gemm, 0)
sqrsum = tl.load(
sqrsum_ptr + ks * sqrsum_stride_k + token_idx * sqrsum_stride_t,
mask=kmask,
other=0.0,
).to(tl.float32)
rstd = tl.rsqrt(tl.sum(sqrsum, 0) * inv_hc_hidden + rms_eps)
scale = tl.load(hc_scale_ptr).to(tl.float32)
base = tl.load(hc_base_ptr + js, mask=jmask, other=0.0).to(tl.float32)
pre = tl.sigmoid(mixes * rstd * scale + base) + hc_pre_eps
tl.store(out_ptr + token_idx * out_stride_t + js * out_stride_j, pre, mask=jmask)
def mhc_pre_mix_triton(
gemm_out: Tensor,
sqrsum: Tensor,
hc_scale: Tensor,
hc_base: Tensor,
hc_mult: int,
hc_hidden_size: int,
rms_eps: float,
hc_pre_eps: float,
) -> Tensor:
"""Pre-mix gate for the delayed mHC pre, from AITER's split-k GEMM output.
AITER's ``mhc_pre_big_fuse`` consumes the unreduced ``[splitk, tokens,
hc_mult3]`` GEMM output and the matching row square-sums, but only returns
the post and comb gates. The delayed formulation also needs the pre gate,
to carry into the next sublayer seam. It is the same slice of the same
numbers, so recover it here rather than repeating the projection.
"""
assert gemm_out.ndim == 3 and gemm_out.dtype == torch.float32
assert sqrsum.ndim == 2 and sqrsum.dtype == torch.float32
splitk, num_tokens = gemm_out.shape[0], gemm_out.shape[1]
assert sqrsum.shape == (splitk, num_tokens)
out = torch.empty(num_tokens, hc_mult, dtype=torch.float32, device=gemm_out.device)
if num_tokens == 0:
return out
_mhc_pre_mix_kernel[(num_tokens,)](
gemm_out,
sqrsum,
hc_scale,
hc_base,
out,
splitk,
hc_mult,
gemm_out.stride(0),
gemm_out.stride(1),
gemm_out.stride(2),
sqrsum.stride(0),
sqrsum.stride(1),
out.stride(0),
out.stride(1),
1.0 / hc_hidden_size,
rms_eps,
hc_pre_eps,
SPLITK_BLOCK=triton.next_power_of_2(splitk),
HC_BLOCK=triton.next_power_of_2(hc_mult),
num_warps=1,
# Match the separate FP32 multiply and add in the Torch reference.
enable_fp_fusion=False,
)
return out
def _mhc_pre_mix_triton_fake(
gemm_out: Tensor,
sqrsum: Tensor,
hc_scale: Tensor,
hc_base: Tensor,
hc_mult: int,
hc_hidden_size: int,
rms_eps: float,
hc_pre_eps: float,
) -> Tensor:
return torch.empty(
gemm_out.shape[1], hc_mult, dtype=torch.float32, device=gemm_out.device
)
def hc_head_reduce_triton_kernel(
x: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
out: torch.Tensor,
norm_eps: float,
hc_eps: float,
) -> None:
x_flat = x.flatten(-2)
x_normed = rmsnorm_nw(x_flat, norm_eps)
mixes = F.linear(x_normed.float(), hc_fn)
pre = torch.sigmoid(mixes * hc_scale + hc_base) + hc_eps
hidden_size = x.shape[-1]
hc_mult = x.shape[-2]
block_h = 1024
_hc_head_reduce_store_kernel[(x.shape[0], (hidden_size + block_h - 1) // block_h)](
pre,
x,
out,
hidden_size,
hc_mult,
pre.stride(0),
pre.stride(1),
x.stride(0),
x.stride(1),
x.stride(2),
out.stride(0),
out.stride(1),
BLOCK_H=block_h,
num_warps=4,
)
def _hc_head_triton(
hs_flat: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
out: torch.Tensor,
hidden_size: int,
rms_eps: float,
hc_eps: float,
hc_mult: int,
) -> None:
"""Fill pre-allocated `out` (T, H) in-place with the hc_head result."""
if hs_flat.shape[0] == 0:
return
hc_head_reduce_triton_kernel(
hs_flat,
fn,
hc_scale,
hc_base,
out,
rms_eps,
hc_eps,
)
return
direct_register_custom_op(
op_name="hc_head_triton",
op_func=_hc_head_triton,
mutates_args=["out"],
)
direct_register_custom_op(
op_name="hc_collapse_triton",
op_func=hc_collapse_triton,
mutates_args=[],
fake_impl=_hc_collapse_triton_fake,
)
direct_register_custom_op(
op_name="mhc_pre_mix_triton",
op_func=mhc_pre_mix_triton,
mutates_args=[],
fake_impl=_mhc_pre_mix_triton_fake,
)
-101
View File
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from dataclasses import asdict, dataclass
from typing import Any
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
compute_num_split,
mhc_pre_big_fuse_with_norm_tilelang,
)
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.math_utils import cdiv
def compute_mhc_pre_num_splits(input_size: int, num_tokens: int) -> int:
splits = compute_num_split(64, input_size, cdiv(num_tokens, 64))
# Bound both GEMM and fused-normalization specializations during startup.
return 1 if splits == 1 else 4 if splits <= 4 else 16
class MHCPreNormKernel(VllmJitKernel["MHCPreNormKernel.CompileKey"]):
@dataclass(frozen=True)
class CompileKey:
hidden_size: int
rms_eps: float
hc_pre_eps: float
hc_sinkhorn_eps: float
hc_post_mult_value: float
sinkhorn_repeat: int
norm_eps: float
n_splits: int
hc_mult: int
use_pre_mix_in: bool
rms_numel: int
save_pre_mix: bool = True
split_mode: str = "fused"
write_aux: bool = False
kernel: Any = staticmethod(mhc_pre_big_fuse_with_norm_tilelang)
def dispatch(self, *, n_splits, **fields) -> CompileKey: # type: ignore[override]
return self.CompileKey(n_splits=n_splits, **fields)
def get_warmup_keys(
self,
*,
max_tokens: int,
extra_splits: Iterable[int] = (),
**fields,
) -> list[CompileKey]:
# The split heuristic changes only at 64-token boundaries. Callers that
# reach this kernel through another GEMM (the fused post + pre-norm
# GEMM) pass the splits that GEMM picks in extra_splits.
splits = sorted(
set(extra_splits)
| (
{
compute_mhc_pre_num_splits(fields["rms_numel"], num_tokens)
for num_tokens in range(1, max_tokens + 1, 64)
}
if is_deep_gemm_supported()
else {1}
)
)
return self._trace_dispatch(self.dispatch)(n_splits=splits, **fields)
def compile(self, compile_key: CompileKey) -> None:
if compile_key not in self._compiled_cache:
self._compiled_cache[compile_key] = self.kernel.compile(
**asdict(compile_key)
)
def compile_many(self, compile_keys: Iterable[CompileKey]) -> None:
missing = list(
dict.fromkeys(
key for key in compile_keys if key not in self._compiled_cache
)
)
if len(missing) < 2:
return super().compile_many(missing)
# TileLang elaborates serially before compiling independent TIR modules.
compiled = self.kernel.par_compile(
[asdict(key) for key in missing], num_workers=min(4, len(missing))
)
self._compiled_cache.update(zip(missing, compiled, strict=True))
def __call__(self, *tensors, **fields):
compile_key = self.dispatch(
n_splits=tensors[0].shape[0],
**fields,
)
if not current_platform.is_cuda():
return self.kernel(*tensors, **asdict(compile_key))
return self._get_or_compile(compile_key)(*tensors)
MHC_PRE_NORM_KERNEL = MHCPreNormKernel()
@@ -1,68 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import cutlass
import cutlass.cute as cute
from vllm.vllm_flash_attn.cute import utils
@cute.jit
def dense_mask_mod(
batch: cute.TensorSSA,
head: cute.TensorSSA,
q_idx: cute.TensorSSA,
kv_idx: cute.TensorSSA,
seqlen_info,
aux_tensors: list,
) -> cute.TensorSSA:
dense_mask = aux_tensors[0]
batch_idx = utils.ssa_to_scalar(batch)
q_idx = utils.ssa_to_scalar(q_idx)
kv_idx = utils.ssa_to_scalar(kv_idx)
batch_stride, query_stride, _ = dense_mask.stride
aligned_mask = cute.make_tensor(
dense_mask.iterator,
cute.make_layout(
dense_mask.shape,
stride=(
cute.assume(batch_stride, divby=4),
cute.assume(query_stride, divby=4),
1,
),
),
)
mask_row = aligned_mask[batch_idx, q_idx, None]
mask_chunks = cute.flat_divide(mask_row, (4,))
mask_chunk = mask_chunks[None, (kv_idx >> 5) >> 2]
loaded = cute.make_rmem_tensor_like(mask_chunk)
cute.autovec_copy(mask_chunk, loaded)
return cute.recast_tensor(loaded, cutlass.Uint32).load()
dense_mask_mod.__vec_size__ = 128
@cute.jit
def offset_dense_mask_mod(
batch: cute.TensorSSA,
head: cute.TensorSSA,
q_idx: cute.TensorSSA,
kv_idx: cute.TensorSSA,
seqlen_info,
aux_tensors: list,
) -> cute.TensorSSA:
dense_mask = aux_tensors[0]
batch_idx = utils.ssa_to_scalar(batch)
q_idx = utils.ssa_to_scalar(q_idx)
key_start = dense_mask[batch_idx, 0, dense_mask.shape[2] - 1]
kv_idx = utils.ssa_to_scalar(kv_idx) + key_start
word_idx = kv_idx >> 5
bit_idx = cutlass.Uint32(kv_idx & 31)
word = dense_mask[batch_idx, q_idx, word_idx]
result = cute.make_rmem_tensor(1, dtype=cutlass.Uint32)
result[0] = utils.shr_u32(cutlass.Uint32(word), bit_idx)
return result.load()
offset_dense_mask_mod.__vec_size__ = 32
@@ -119,11 +119,6 @@ if HAS_TRITON:
TritonExperts,
TritonWNA16Experts,
)
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExperts,
XPUExpertsFp8,
XPUExpertsMxFp4,
)
from vllm.model_executor.layers.fused_moe.fused_moe import (
fused_experts,
get_config_file_name,
File diff suppressed because it is too large Load Diff
@@ -1,353 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
kInt4Static,
kInt4Static32,
kMxfp4Dynamic,
kMxfp4Static,
kMxfp8Dynamic,
kMxfp8Static,
)
from vllm.platforms import current_platform
if current_platform.is_xpu():
from vllm_xpu_kernels.fused_moe_interface import XpuFusedMoe
def prepare_fp8_moe_layer_for_xpu(
w13: torch.Tensor,
w13_scale: torch.Tensor,
w2: torch.Tensor,
w2_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
if w13_scale is not None and w13_scale.ndim == 3:
w13_scale = w13_scale.transpose(-1, -2).contiguous()
if w2_scale is not None and w2_scale.ndim == 3:
w2_scale = w2_scale.transpose(-1, -2).contiguous()
return (
w13.transpose(-1, -2).contiguous(),
w13_scale,
w2.transpose(-1, -2).contiguous(),
w2_scale,
)
class XPUExperts(mk.FusedMoEExpertsModular):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
self.fused_moe_impl: XpuFusedMoe | None = None
is_xe2_or_xe3 = torch.ops._xpu_C.is_xe2_arch() or torch.ops._xpu_C.is_xe3_arch()
if not is_xe2_or_xe3:
raise NotImplementedError(
"XPUExperts is only supported on Intel Xe2/Xe3 GPUs"
)
self._expects_unquantized_inputs = is_xe2_or_xe3
@property
def expects_unquantized_inputs(self) -> bool:
return self._expects_unquantized_inputs
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_xpu()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return True
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.RELU2_NO_MUL,
]
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
return True
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(None, None),
(kFp8StaticTensorSym, None),
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
workspace1 = (0,)
workspace2 = (0,)
output = (M, K)
return (workspace1, workspace2, output)
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
if self.fused_moe_impl is None:
topk = topk_ids.size(-1)
if (
self.quant_config is not None
and self.quant_config.weight_quant_dtype == "mxfp4"
):
w1 = w1.view(torch.float4_e2m1fn_x2)
w2 = w2.view(torch.float4_e2m1fn_x2)
self.fused_moe_impl = XpuFusedMoe(
w13=w1,
w13_scales=self.w1_scale,
w13_bias=self.w1_bias,
w2=w2,
w2_scales=self.w2_scale,
w2_bias=self.w2_bias,
n_experts_per_token=topk,
activation=activation.value,
num_experts=self.moe_config.num_local_experts,
ep_rank=self.moe_config.ep_rank,
ep_size=self.moe_config.ep_size,
expert_map=expert_map,
gemm1_clamp_limit=self.gemm1_clamp_limit,
)
assert self.fused_moe_impl is not None
if self.moe_config.ep_size > 1:
# Avoid topk_ids = -1, remap them to an
# expert not owned by this rank so they are skipped.
sentinel = global_num_experts - 1 if self.moe_config.ep_rank == 0 else 0
topk_ids = torch.where(topk_ids < 0, sentinel, topk_ids)
self.fused_moe_impl.apply(
output=output,
hidden_states=hidden_states,
topk_weights=topk_weights,
topk_ids=topk_ids,
a1q_scale=a1q_scale,
)
class XPUExpertsFp8(XPUExperts):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kFp8StaticTensorSym, None),
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
class XPUExpertsMxFp8(XPUExpertsFp8):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
assert quant_config.quant_dtype == "mxfp8"
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kMxfp8Static, None),
(kMxfp8Static, kMxfp8Dynamic),
]
return (weight_key, activation_key) in SUPPORTED_W_A
class XPUExpertsBlockFp8(XPUExperts):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
class XPUExpertsWNA16(XPUExperts):
"""W4A16 INT4-symmetric MoE backed by `xpu_fused_moe(is_int4=True)`.
Weight layout when `is_int4=True` (per `xpu_fused_moe` docstring):
w13: [num_experts, 2*inter_size, hidden_size] contiguous int4-packed
w13_scales: [num_experts, 2*inter_size, hidden_size // group_size]
w2: [num_experts, hidden_size, inter_size] contiguous int4-packed
w2_scales: [num_experts, hidden_size, inter_size // group_size]
Pairs with `INCXPULinearMethod` for the linear layers; together they
cover full-attn + MoE on Intel XPU end-to-end without IPEX.
"""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) in (
(kInt4Static, None),
(kInt4Static32, None),
)
class XPUExpertsMxFp4(XPUExperts):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
max_num_tokens,
num_dispatchers,
)
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# K = a1q.size(-1). When activations are pre-quantized packed mxfp4,
# K is the packed hidden_size (= logical / 2); the kernel output is at
# logical hidden_size (2 * K). When unquantized (bf16), K is already
# the logical size.
logical_K = K if self.expects_unquantized_inputs else 2 * K
return (0,), (0,), (M, logical_K)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kMxfp4Static, None),
(kMxfp4Static, kMxfp4Dynamic),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@@ -241,21 +241,8 @@ def backend_to_kernel_cls(
return [CutlassExpertsMxfp4]
elif backend == Mxfp4MoeBackend.TRITON:
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
OAITritonExperts,
OAITritonMxfp4ExpertsMonolithic,
)
# NOTE: prefer Monolithic > Modular, so return Monolithic first.
return [OAITritonMxfp4ExpertsMonolithic, OAITritonExperts]
elif backend == Mxfp4MoeBackend.TRITON_UNFUSED:
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
UnfusedOAITritonExperts,
)
return [UnfusedOAITritonExperts]
elif backend in (Mxfp4MoeBackend.TRITON, Mxfp4MoeBackend.TRITON_UNFUSED):
raise ImportError("gpt_oss triton_kernels experts were removed in this build.")
elif backend == Mxfp4MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
@@ -1,213 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Producer side of the QuantizedActivation contract for activation layers.
Given an activation module and the downstream linear it feeds, fuse the
activation with that linear's input quantization into a single kernel when the
linear advertises a consumable input quantization key (see quant_activation.py).
Falls back to the plain activation when nothing matches, so a model forward can
always call maybe_fused_act_quant unconditionally.
This is the manual-fusion counterpart to ActivationQuantFusionPass: when fusion
fires here, the activation and quantization are already consumed, so a compiler
pass cannot fuse the same boundary again.
"""
from collections.abc import Callable
import torch
from vllm.model_executor.layers.activation import ReLUSquaredActivation, SiluAndMul
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
get_input_quant_key,
)
from vllm.model_executor.layers.fusion.relu2_fp8_quant import (
relu_squared_static_fp8_quant,
)
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
FP8_DTYPE = current_platform.fp8_dtype()
FP4_DTYPE = torch.uint8
def _silu_and_mul_fp8_static(
x: torch.Tensor, linear: LinearBase
) -> QuantizedActivation:
"""SiluAndMul + FP8 static per-tensor quantization."""
d = x.shape[-1] // 2
out_shape = x.shape[:-1] + (d,)
result = torch.empty(out_shape, dtype=FP8_DTYPE, device=x.device)
# TODO(mgoin): read the consumer scale via the contract instead of reaching
# into the kernel-specific input_scale attribute.
scale = linear.input_scale
torch.ops._C.silu_and_mul_quant(result, x, scale)
return QuantizedActivation(
data=result,
scale=scale,
orig_dtype=x.dtype,
orig_shape=out_shape,
quant_key=kFp8StaticTensorSym,
)
def _silu_and_mul_fp8_dynamic_block(
x: torch.Tensor, linear: LinearBase, group_size: int, quant_key: QuantKey
) -> QuantizedActivation:
"""SiluAndMul + FP8 dynamic per-block quantization."""
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
d = x.shape[-1] // 2
out_shape = x.shape[:-1] + (d,)
x = x.reshape(-1, x.shape[-1])
num_tokens = x.shape[0]
num_groups = d // group_size
result = torch.empty((num_tokens, d), dtype=FP8_DTYPE, device=x.device)
scales = torch.empty((num_tokens, num_groups), dtype=torch.float32, device=x.device)
torch.ops._C.silu_and_mul_per_block_quant(
out=result,
input=x,
scales=scales,
group_size=group_size,
scale_ub=None,
is_scale_transposed=False,
)
return QuantizedActivation(
data=result.view(out_shape),
scale=scales.view(out_shape[:-1] + (num_groups,)),
orig_dtype=x.dtype,
orig_shape=out_shape,
quant_key=quant_key,
)
def _silu_and_mul_fp8_dynamic_128(
x: torch.Tensor, linear: LinearBase
) -> QuantizedActivation:
"""SiluAndMul + FP8 dynamic per-group (group=128) quantization."""
return _silu_and_mul_fp8_dynamic_block(x, linear, 128, kFp8Dynamic128Sym)
def _silu_and_mul_nvfp4_dynamic(
x: torch.Tensor, linear: LinearBase
) -> QuantizedActivation:
"""SiluAndMul + NVFP4 dynamic quantization."""
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
d = x.shape[-1] // 2
out_shape = x.shape[:-1] + (d,)
x = x.reshape(-1, x.shape[-1])
num_tokens = x.shape[0]
# NVFP4 packs 2 values into 1 byte
result = torch.empty((num_tokens, d // 2), dtype=FP4_DTYPE, device=x.device)
# Block scale output: swizzled tensor-core layout
# [num_m_tiles, num_k_tiles, 32, 4, 4] of int32-packed FP8 scales, so the
# row/col extents must be padded to the 128x4 tile like
# scaled_fp4_quant's allocator (create_fp4_scale_tensor). Each group of 16
# elements shares one FP8 scale.
rounded_m = round_up(num_tokens, 128)
rounded_n = round_up(d // 16, 4)
block_scale = torch.empty(
(rounded_m, rounded_n // 4), dtype=torch.int32, device=x.device
).view(FP8_DTYPE)
# The kernel folds the global scale into the block scales, and the
# consumer GEMM's alpha (= input_global_scale * weight_global_scale)
# divides it back out, so quantize with the reciprocal like the unfused
# scaled_fp4_quant path does.
input_global_scale_inv = getattr(linear, "input_global_scale_inv", None)
assert input_global_scale_inv is not None, (
"input_global_scale_inv is required for NVFP4 quantization"
)
torch.ops._C.silu_and_mul_nvfp4_quant(
result, block_scale, x, input_global_scale_inv
)
return QuantizedActivation(
data=result.view(out_shape[:-1] + (d // 2,)),
scale=block_scale,
orig_dtype=x.dtype,
orig_shape=out_shape,
quant_key=kNvfp4Dynamic,
)
# (activation module type, consumer input quantization key) -> fused producer.
# Add a row for each supported manual producer.
_FUSED_ACT_QUANT: dict[tuple[type, QuantKey], Callable] = {
(SiluAndMul, kFp8StaticTensorSym): _silu_and_mul_fp8_static,
}
def _relu_squared_static_fp8_quant_supported(
act_fn: torch.nn.Module,
x: torch.Tensor,
linear: LinearBase,
) -> bool:
"""Return whether the ReLU2 static-FP8 producer can consume this input."""
scale = getattr(linear, "input_scale", None)
return (
isinstance(act_fn, ReLUSquaredActivation)
and x.is_cuda
and x.dtype == torch.bfloat16
and x.is_contiguous()
and isinstance(scale, torch.Tensor)
and scale.dtype == torch.float32
and scale.device == x.device
and scale.numel() == 1
)
# Optional per-entry predicates preserve the unconditional fallback contract
# for producers whose supported inputs are narrower than their registry key.
_FUSED_ACT_QUANT_SUPPORT: dict[
tuple[type, QuantKey], Callable[[torch.nn.Module, torch.Tensor, LinearBase], bool]
] = {}
# This path is validated on SM90 and newer CUDA devices.
if current_platform.is_cuda() and current_platform.has_device_capability(90):
_RELU2_STATIC_FP8_KEY = (ReLUSquaredActivation, kFp8StaticTensorSym)
_FUSED_ACT_QUANT[_RELU2_STATIC_FP8_KEY] = relu_squared_static_fp8_quant
_FUSED_ACT_QUANT_SUPPORT[_RELU2_STATIC_FP8_KEY] = (
_relu_squared_static_fp8_quant_supported
)
# Add CUDA-specific entries for dynamic block quantization
if current_platform.is_cuda_alike():
_FUSED_ACT_QUANT[(SiluAndMul, kFp8Dynamic128Sym)] = _silu_and_mul_fp8_dynamic_128
# Add NVFP4 if supported (requires SM100+)
if current_platform.is_cuda() and hasattr(torch.ops._C, "silu_and_mul_nvfp4_quant"):
_FUSED_ACT_QUANT[(SiluAndMul, kNvfp4Dynamic)] = _silu_and_mul_nvfp4_dynamic
def maybe_fused_act_quant(
act_fn: torch.nn.Module,
x: torch.Tensor,
linear: LinearBase,
) -> "torch.Tensor | QuantizedActivation":
"""Apply act_fn, fusing the downstream linear's input quant when possible.
Returns a QuantizedActivation when a fused kernel matches the activation and
the consumer's effective input quantization key, else the plain activation.
"""
key = get_input_quant_key(linear)
if key is not None:
registry_key = (type(act_fn), key)
producer = _FUSED_ACT_QUANT.get(registry_key)
support = _FUSED_ACT_QUANT_SUPPORT.get(registry_key)
if producer is not None and (support is None or support(act_fn, x, linear)):
return producer(x, linear)
return act_fn(x)
@@ -1,14 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointExporter,
)
def kda_prefill_checkpoint_alignment(backend: str) -> int | None:
return 16 if backend == "flashkda" else None
class FlashKDAPrefillCheckpointExporter(MambaPrefillCheckpointExporter):
"""Store FlashKDA recurrent and convolution checkpoint states."""
@@ -1,704 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import torch
import torch.nn.functional as F
from transformers.configuration_utils import PreTrainedConfig
from vllm.config import (
VllmConfig,
get_current_vllm_config,
)
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.mamba.linear.base import LinearAttention
from vllm.model_executor.layers.mamba.linear.minimax_linear_attn import (
MiniMaxText01LinearAttention,
MiniMaxText01LinearKernel,
linear_attention_decode,
)
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
RMSNormGated,
layernorm_fn,
)
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
def _build_rope_parameters(config: PreTrainedConfig) -> dict | None:
rope_parameters = copy.deepcopy(getattr(config, "rope_parameters", None)) or {}
if "rope_theta" not in rope_parameters and hasattr(config, "rope_theta"):
rope_parameters["rope_theta"] = config.rope_theta
if "partial_rotary_factor" not in rope_parameters and hasattr(
config, "partial_rotary_factor"
):
rope_parameters["partial_rotary_factor"] = config.partial_rotary_factor
rope_scaling = getattr(config, "rope_scaling", None)
if isinstance(rope_scaling, dict):
rope_scaling = copy.deepcopy(rope_scaling)
if "type" in rope_scaling and "rope_type" not in rope_scaling:
rope_scaling["rope_type"] = rope_scaling.pop("type")
rope_parameters.update(rope_scaling)
return rope_parameters or None
def clear_linear_attention_cache_for_new_sequences(
kv_cache: torch.Tensor,
state_indices_tensor: torch.Tensor,
attn_metadata: LinearAttentionMetadata,
) -> None:
num_prefills = getattr(attn_metadata, "num_prefills", 0)
if num_prefills <= 0:
return
num_decodes = getattr(attn_metadata, "num_decodes", 0)
prefill_state_indices = getattr(attn_metadata, "state_indices_tensor_p", None)
for prefill_idx in range(num_prefills):
if num_decodes + prefill_idx + 1 >= len(attn_metadata.query_start_loc):
break
q_start = attn_metadata.query_start_loc[num_decodes + prefill_idx]
q_end = attn_metadata.query_start_loc[num_decodes + prefill_idx + 1]
query_len = q_end - q_start
context_len = attn_metadata.seq_lens[num_decodes + prefill_idx] - query_len
if context_len == 0:
if prefill_state_indices is not None:
block_to_clear = prefill_state_indices[prefill_idx]
else:
block_to_clear = state_indices_tensor[num_decodes + prefill_idx]
kv_cache[block_to_clear, ...] = 0
def linear_attention_prefill_and_mix(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_cache: torch.Tensor,
state_indices_tensor: torch.Tensor,
attn_metadata: LinearAttentionMetadata,
slope_rate: torch.Tensor,
block_size: int,
decode_fn,
prefix_fn,
layer_idx: int | None = None,
) -> torch.Tensor:
hidden = []
req_offset = getattr(attn_metadata, "num_decodes", 0)
prefill_state_indices = getattr(attn_metadata, "state_indices_tensor_p", None)
for _prefill_idx in range(getattr(attn_metadata, "num_prefills", 0)):
if req_offset + _prefill_idx + 1 >= len(attn_metadata.query_start_loc):
break
if prefill_state_indices is not None and _prefill_idx >= len(
prefill_state_indices
):
break
if prefill_state_indices is None and _prefill_idx >= len(state_indices_tensor):
break
_start = attn_metadata.query_start_loc[req_offset + _prefill_idx]
_end = attn_metadata.query_start_loc[req_offset + _prefill_idx + 1]
if prefill_state_indices is not None:
slot_id = prefill_state_indices[_prefill_idx]
else:
slot_id = state_indices_tensor[req_offset + _prefill_idx]
qs = q[_start:_end].transpose(0, 1).contiguous()
ks = k[_start:_end].transpose(0, 1).contiguous()
vs = v[_start:_end].transpose(0, 1).contiguous()
slice_layer_cache = kv_cache[slot_id, ...]
out_slice = prefix_fn(
qs,
ks,
vs,
slice_layer_cache,
slope_rate,
block_size,
layer_idx=layer_idx,
)
hidden.append(out_slice.contiguous())
if attn_metadata.num_decode_tokens > 0:
hidden_decode = decode_fn(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
hidden.insert(0, hidden_decode)
if not hidden:
return torch.empty((0, q.size(1) * q.size(2)), device=q.device, dtype=q.dtype)
hidden = torch.concat(hidden, dim=0).contiguous()
return hidden
@triton.jit
def _bailing_linear_attn_decode_spec_step_kernel(
q_ptr,
k_ptr,
v_ptr,
kv_cache_ptr,
slope_rate_ptr,
state_indices_ptr,
query_start_loc_ptr,
num_accepted_tokens_ptr,
output_ptr,
q_start: tl.constexpr,
D: tl.constexpr,
q_b_stride,
q_h_stride,
q_d_stride,
k_b_stride,
k_h_stride,
k_d_stride,
v_b_stride,
v_h_stride,
v_d_stride,
cache_b_stride,
cache_h_stride,
cache_d0_stride,
cache_d1_stride,
state_indices_b_stride,
state_indices_t_stride,
output_b_stride,
output_d_stride,
DRAFT_IDX: tl.constexpr,
STATE_WIDTH: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
req_id = tl.program_id(0)
head_id = tl.program_id(1)
block_id = tl.program_id(2)
req_start = tl.load(query_start_loc_ptr + req_id).to(tl.int64)
req_end = tl.load(query_start_loc_ptr + req_id + 1).to(tl.int64)
query_len = req_end - req_start
if query_len <= DRAFT_IDX:
return
dst_slot = tl.load(
state_indices_ptr
+ req_id * state_indices_b_stride
+ DRAFT_IDX * state_indices_t_stride
).to(tl.int64)
if dst_slot == -1:
return
if DRAFT_IDX == 0:
accepted_offset = tl.load(num_accepted_tokens_ptr + req_id).to(tl.int64) - 1
accepted_offset = tl.maximum(accepted_offset, 0)
accepted_offset = tl.minimum(accepted_offset, STATE_WIDTH - 1)
src_slot = tl.load(
state_indices_ptr
+ req_id * state_indices_b_stride
+ accepted_offset * state_indices_t_stride
).to(tl.int64)
else:
src_slot = tl.load(
state_indices_ptr
+ req_id * state_indices_b_stride
+ (DRAFT_IDX - 1) * state_indices_t_stride
).to(tl.int64)
if src_slot == -1:
return
token_idx = req_start - q_start + DRAFT_IDX
qk_offsets = tl.arange(0, D)
v_offsets = tl.arange(0, BLOCK_SIZE) + block_id * BLOCK_SIZE
qk_mask = qk_offsets < D
v_mask = v_offsets < D
kv_mask = qk_mask[:, None] & v_mask[None, :]
q = tl.load(
q_ptr + token_idx * q_b_stride + head_id * q_h_stride + qk_offsets * q_d_stride,
mask=qk_mask,
other=0.0,
)
k = tl.load(
k_ptr + token_idx * k_b_stride + head_id * k_h_stride + qk_offsets * k_d_stride,
mask=qk_mask,
other=0.0,
)
v = tl.load(
v_ptr + token_idx * v_b_stride + head_id * v_h_stride + v_offsets * v_d_stride,
mask=v_mask,
other=0.0,
)
cache_offsets = (
qk_offsets[:, None] * cache_d0_stride + v_offsets[None, :] * cache_d1_stride
)
src_cache_ptr = (
kv_cache_ptr
+ src_slot * cache_b_stride
+ head_id * cache_h_stride
+ cache_offsets
)
dst_cache_ptr = (
kv_cache_ptr
+ dst_slot * cache_b_stride
+ head_id * cache_h_stride
+ cache_offsets
)
slope = tl.load(slope_rate_ptr + head_id)
decay = tl.exp(-slope)
kv_old = tl.load(src_cache_ptr, mask=kv_mask, other=0.0)
kv_new = k[:, None] * v[None, :] + decay * kv_old
output = tl.sum(q[:, None].to(tl.float32) * kv_new, axis=0)
tl.store(dst_cache_ptr, kv_new, mask=kv_mask)
tl.store(
output_ptr
+ token_idx * output_b_stride
+ (head_id * D + v_offsets) * output_d_stride,
output,
mask=v_mask,
)
def bailing_linear_attention_decode_spec(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_cache: torch.Tensor,
slope_rate: torch.Tensor,
state_indices_tensor: torch.Tensor,
query_start_loc: torch.Tensor,
num_accepted_tokens: torch.Tensor,
q_start: int,
q_end: int | None,
slot_start: int,
slot_end: int | None,
block_size: int,
) -> torch.Tensor:
q_decode = q[q_start:q_end]
k_decode = k[q_start:q_end]
v_decode = v[q_start:q_end]
hidden = torch.empty(
(q_decode.shape[0], q.shape[1] * q.shape[2]),
device=q.device,
dtype=q.dtype,
)
hidden.zero_()
state_indices_tensor = state_indices_tensor[slot_start:slot_end]
query_start_loc = query_start_loc.to(device=q.device)
batch_size = state_indices_tensor.shape[0]
num_heads = q_decode.shape[1]
head_dim = q_decode.shape[2]
assert k_decode.shape == (q_decode.shape[0], num_heads, head_dim)
assert v_decode.shape == (q_decode.shape[0], num_heads, head_dim)
state_width = state_indices_tensor.shape[1]
grid = (batch_size, num_heads, triton.cdiv(head_dim, block_size))
for draft_idx in range(state_width):
_bailing_linear_attn_decode_spec_step_kernel[grid](
q_decode,
k_decode,
v_decode,
kv_cache,
slope_rate,
state_indices_tensor,
query_start_loc,
num_accepted_tokens[:batch_size],
hidden,
q_start,
head_dim,
q_decode.stride(0),
q_decode.stride(1),
q_decode.stride(2),
k_decode.stride(0),
k_decode.stride(1),
k_decode.stride(2),
v_decode.stride(0),
v_decode.stride(1),
v_decode.stride(2),
kv_cache.stride(0),
kv_cache.stride(1),
kv_cache.stride(2),
kv_cache.stride(3),
state_indices_tensor.stride(0),
state_indices_tensor.stride(1),
hidden.stride(0),
hidden.stride(1),
DRAFT_IDX=draft_idx,
STATE_WIDTH=state_width,
BLOCK_SIZE=block_size,
)
return hidden
class BailingGroupRMSNormGate(RMSNormGated):
def __init__(
self,
hidden_size,
eps=1e-5,
group_size=None,
norm_before_gate=True,
device=None,
dtype=None,
):
super().__init__(
hidden_size,
eps=eps,
group_size=group_size,
norm_before_gate=norm_before_gate,
device=device,
dtype=dtype,
activation="sigmoid",
)
# Add custom weight loader for TP sharding
self.weight.weight_loader = self._weight_loader
@staticmethod
def _weight_loader(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None:
"""Load weight with TP sharding."""
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
shard_size = loaded_weight.shape[0] // tp_size
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
param.data.copy_(loaded_weight[shard].contiguous())
# --8<-- [start:bailing_moe_linear_attention]
@PluggableLayer.register("bailing_moe_linear_attention")
class BailingMoELinearAttention(LinearAttention):
"""Pluggable Bailing MoE Linear Attention layer which allows OOT backends
to add custom implementations.
This implements the linear attention mechanism from sglang, adapted for
vLLM's v1 engine with MambaBase interface support.
"""
# --8<-- [end:bailing_moe_linear_attention]
def __init__(
self,
config: PreTrainedConfig,
vllm_config: VllmConfig,
prefix: str = "linear_attn",
):
super().__init__(config, vllm_config, prefix)
self.scaling = self.head_dim**-0.5
self.tp_heads = self.num_heads // self.tp_size
self.max_position_embeddings = config.max_position_embeddings
self.rope_theta = getattr(config, "rope_theta", 600000)
self.tp_kv_heads = self.num_heads // self.tp_size
self.q_size_per_rank = self.head_dim * self.tp_heads
self.kv_size_per_rank = self.head_dim * self.tp_kv_heads
self.use_qk_norm = getattr(config, "use_qk_norm", False)
self.linear_backend = "minimax"
self.linear_scale = self.linear_backend == "minimax"
self.linear_rope = getattr(config, "linear_rope", True)
if hasattr(config, "use_linear_silu"):
self.linear_silu = config.use_linear_silu
elif hasattr(config, "linear_silu"):
self.linear_silu = config.linear_silu
else:
self.linear_silu = False
self.query_key_value = QKVParallelLinear(
self.hidden_size,
self.head_dim,
self.num_heads,
self.num_heads, # MHA: kv_heads = num_heads
bias=(config.use_bias or config.use_qkv_bias),
quant_config=self.quant_config,
prefix=f"{prefix}.query_key_value",
)
if self.use_qk_norm:
self.query_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.key_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.g_proj = ColumnParallelLinear(
self.hidden_size,
self.hidden_inner_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.g_proj",
)
self.dense = RowParallelLinear(
self.hidden_inner_size,
self.hidden_size,
bias=config.use_bias,
quant_config=self.quant_config,
prefix=f"{prefix}.dense",
reduce_results=True,
)
self.group_norm_size = getattr(config, "group_norm_size", 1)
self.rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
assert self.tp_size <= self.group_norm_size, (
"tp_size must be <= group_norm_size for local rms norm"
)
assert self.group_norm_size % self.tp_size == 0, (
"group_norm_size must be divisible by tp_size"
)
# When group_norm_size == 1, group_size equals hidden_size // tp_size
self.g_norm = BailingGroupRMSNormGate(
hidden_size=self.hidden_inner_size // self.tp_size,
eps=self.rms_norm_eps,
group_size=(
self.hidden_inner_size // self.group_norm_size
if self.group_norm_size > 1
else self.hidden_inner_size // self.tp_size
),
)
# use fp32 rotary embedding
rope_parameters = _build_rope_parameters(config)
self.rotary_emb = get_rope(
self.head_dim,
max_position=self.max_position_embeddings,
is_neox_style=True,
rope_parameters=rope_parameters or None,
)
# Build slope tensor for linear attention decay
slope_rate = MiniMaxText01LinearAttention._build_slope_tensor(self.num_heads)
if self.num_hidden_layers <= 1:
self.slope_rate = slope_rate * (1 + 1e-5)
else:
self.slope_rate = slope_rate * (
1 - self.layer_idx / (self.num_hidden_layers - 1) + 1e-5
)
self.tp_slope = self.slope_rate[
self.tp_rank * self.tp_heads : (self.tp_rank + 1) * self.tp_heads
].contiguous()
# Register for compilation
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def get_attn_backend(self):
from vllm.v1.attention.backends.linear_attn import (
BailingLinearAttentionBackend,
)
return BailingLinearAttentionBackend
@staticmethod
def weight_direct_load(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
"""Load weight for linear attention layers.
For FP8 quantized parameters, we need to use the weight_loader if available,
as it handles special cases like tensor parallelism sharding.
"""
# Check if param has a weight_loader (for vLLM ModelWeightParameter)
weight_loader = getattr(param, "weight_loader", None)
if weight_loader is not None:
# Use the weight_loader which handles TP sharding and quantization
weight_loader(param, loaded_weight)
else:
# Fall back to direct copy for standard tensors
assert param.size() == loaded_weight.size(), (
f"Shape mismatch: {param.shape} vs {loaded_weight.shape}"
)
param.data.copy_(loaded_weight)
def forward(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
positions: torch.Tensor,
) -> None:
"""Forward method called by torch.ops.vllm.linear_attention."""
torch.ops.vllm.linear_attention(
hidden_states,
output,
positions,
self.prefix,
)
def _forward(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
positions: torch.Tensor,
) -> None:
"""Actual forward implementation."""
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
if attn_metadata is not None:
assert isinstance(attn_metadata, dict)
attn_metadata = attn_metadata.get(self.prefix) # type: ignore
if attn_metadata is not None:
assert isinstance(attn_metadata, LinearAttentionMetadata)
num_actual_tokens = (
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
)
else:
num_actual_tokens = hidden_states.shape[0]
# QKV projection
qkv, _ = self.query_key_value(hidden_states[:num_actual_tokens])
# use rotary_emb support fp32
qkv = qkv.to(torch.float32)
if self.linear_silu:
qkv = F.silu(qkv)
# Split q, k, v
q, k, v = torch.split(
qkv,
[self.q_size_per_rank, self.kv_size_per_rank, self.kv_size_per_rank],
dim=-1,
)
# Apply QK norm if needed
if self.use_qk_norm:
q = q.reshape(-1, self.tp_heads, self.head_dim)
k = k.reshape(-1, self.tp_kv_heads, self.head_dim)
q = layernorm_fn(
q,
self.query_layernorm.weight.data,
bias=None,
eps=self.rms_norm_eps,
is_rms_norm=True,
)
k = layernorm_fn(
k,
self.key_layernorm.weight.data,
bias=None,
eps=self.rms_norm_eps,
is_rms_norm=True,
)
q = q.reshape(-1, self.q_size_per_rank)
k = k.reshape(-1, self.kv_size_per_rank)
# Apply rotary embeddings
if self.linear_rope:
q, k = self.rotary_emb(positions[:num_actual_tokens], q, k)
# Reshape to [batch, heads, seq_len, head_dim]
q = q.view((qkv.shape[0], self.tp_heads, self.head_dim))
k = k.view((qkv.shape[0], self.tp_kv_heads, self.head_dim))
v = v.view((qkv.shape[0], self.tp_kv_heads, self.head_dim))
# Apply scaling if using minimax backend
if self.linear_scale:
q = q * self.scaling
# Get KV cache and state indices
if attn_metadata is not None:
kv_cache = self.kv_cache[0]
state_indices_tensor = attn_metadata.state_indices_tensor
clear_linear_attention_cache_for_new_sequences(
kv_cache, state_indices_tensor, attn_metadata
)
# Compute attention
decode_only = getattr(attn_metadata, "num_prefills", 0) == 0
if attn_metadata is None:
hidden = torch.empty(
(q.shape[0], q.shape[1] * q.shape[2]), device=q.device, dtype=q.dtype
)
else:
if not decode_only:
hidden = self._prefill_and_mix_infer(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
else:
hidden = self._decode_infer(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
# Apply group norm and gate (matching SGLang behavior)
gate, _ = self.g_proj(hidden_states[:num_actual_tokens])
if self.group_norm_size > 1:
hidden = self.g_norm(hidden, gate)
else:
hidden = self.g_norm(hidden)
hidden = F.sigmoid(gate) * hidden
hidden = hidden.to(hidden_states.dtype)
# Output projection
dense_out, _ = self.dense(hidden)
output[:num_actual_tokens] = dense_out
def _prefill_and_mix_infer(
self, q, k, v, kv_cache, state_indices_tensor, attn_metadata
):
"""Handle prefill (mixed with decode if any)."""
return linear_attention_prefill_and_mix(
q=q,
k=k,
v=v,
kv_cache=kv_cache,
state_indices_tensor=state_indices_tensor,
attn_metadata=attn_metadata,
slope_rate=self.tp_slope,
block_size=self.BLOCK,
decode_fn=self._decode_infer,
prefix_fn=MiniMaxText01LinearKernel.jit_linear_forward_prefix,
layer_idx=self.layer_idx,
)
def _decode_infer(self, q, k, v, kv_cache, state_indices_tensor, attn_metadata):
"""Handle decode (single token per sequence)."""
decode_state_indices = getattr(attn_metadata, "state_indices_tensor_d", None)
num_accepted_tokens = getattr(attn_metadata, "num_accepted_tokens", None)
query_start_loc = getattr(attn_metadata, "query_start_loc_d", None)
if (
decode_state_indices is not None
and decode_state_indices.dim() > 1
and num_accepted_tokens is not None
and query_start_loc is not None
):
return bailing_linear_attention_decode_spec(
q,
k,
v,
kv_cache,
self.tp_slope,
decode_state_indices,
query_start_loc,
num_accepted_tokens,
q_start=0,
q_end=attn_metadata.num_decode_tokens,
slot_start=0,
slot_end=attn_metadata.num_decodes,
block_size=32,
)
decode_state_indices = (
state_indices_tensor
if decode_state_indices is None
else decode_state_indices
)
hidden = linear_attention_decode(
q,
k,
v,
kv_cache,
self.tp_slope,
decode_state_indices,
q_start=0,
q_end=attn_metadata.num_decode_tokens,
slot_start=0,
slot_end=attn_metadata.num_decodes,
block_size=32,
)
return hidden
@@ -1,66 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from transformers import PreTrainedConfig
from vllm.config import (
VllmConfig,
)
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.mamba.abstract import MambaBase
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateDtypeCalculator,
MambaStateShapeCalculator,
)
from vllm.model_executor.models.utils import extract_layer_index
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
class LinearAttention(PluggableLayer, MambaBase):
"""Base class for Linear attention layer."""
def __init__(
self, config: PreTrainedConfig, vllm_config: VllmConfig, prefix: str = ""
):
super().__init__()
self.layer_idx = extract_layer_index(prefix)
self.prefix = prefix
self.model_config = vllm_config.model_config
self.cache_config = vllm_config.cache_config
self.quant_config = vllm_config.quant_config
self.BLOCK = getattr(config, "block", 256)
self.hidden_size = config.hidden_size
self.num_heads = config.num_attention_heads
self.num_hidden_layers = config.num_hidden_layers
self.head_dim = (
config.head_dim
if hasattr(config, "head_dim")
else config.hidden_size // self.num_heads
)
self.hidden_inner_size = self.head_dim * self.num_heads
self.tp_size = get_tensor_model_parallel_world_size()
self.tp_rank = get_tensor_model_parallel_rank()
assert self.num_heads % self.tp_size == 0
@property
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.LINEAR
def get_state_dtype(self) -> tuple[torch.dtype]:
assert self.model_config is not None
assert self.cache_config is not None
return MambaStateDtypeCalculator.linear_attention_state_dtype(
self.model_config.dtype,
self.cache_config.mamba_cache_dtype,
)
def get_state_shape(self) -> tuple[tuple[int, int, int], ...]:
return MambaStateShapeCalculator.linear_attention_state_shape(
num_heads=self.num_heads, tp_size=self.tp_size, head_dim=self.head_dim
)
@@ -1,345 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Callable
import torch
import torch.nn.functional as F
from einops import rearrange
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.lightning_attn import (
lightning_attention,
linear_decode_forward_triton,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.mamba.linear.base import LinearAttention
from vllm.model_executor.layers.minimax_rms_norm import MiniMaxText01RMSNormTP
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
def clear_linear_attention_cache_for_new_sequences(
kv_cache: torch.Tensor,
state_indices_tensor: torch.Tensor,
attn_metadata: LinearAttentionMetadata,
) -> None:
num_prefills = getattr(attn_metadata, "num_prefills", 0)
if num_prefills <= 0:
return
num_decode_tokens = getattr(attn_metadata, "num_decode_tokens", 0)
for prefill_idx in range(num_prefills):
q_start = attn_metadata.query_start_loc[num_decode_tokens + prefill_idx]
q_end = attn_metadata.query_start_loc[num_decode_tokens + prefill_idx + 1]
query_len = q_end - q_start
context_len = (
attn_metadata.seq_lens[num_decode_tokens + prefill_idx] - query_len
)
if context_len == 0:
block_to_clear = state_indices_tensor[num_decode_tokens + prefill_idx]
kv_cache[block_to_clear, ...] = 0
def linear_attention_decode(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_cache: torch.Tensor,
slope_rate: torch.Tensor,
state_indices_tensor: torch.Tensor,
q_start: int = 0,
q_end: int | None = None,
slot_start: int = 0,
slot_end: int | None = None,
block_size: int = 32,
) -> torch.Tensor:
q = q[q_start:q_end].unsqueeze(2).contiguous()
k = k[q_start:q_end].unsqueeze(2).contiguous()
v = v[q_start:q_end].unsqueeze(2).contiguous()
slot_id = state_indices_tensor[slot_start:slot_end]
return linear_decode_forward_triton(
q, k, v, kv_cache, slope_rate, slot_id, block_size
)
def linear_attention_prefill_and_mix(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_cache: torch.Tensor,
state_indices_tensor: torch.Tensor,
attn_metadata: LinearAttentionMetadata,
slope_rate: torch.Tensor,
block_size: int,
decode_fn: Callable[..., torch.Tensor],
prefix_fn: Callable[..., torch.Tensor],
layer_idx: int | None = None,
) -> torch.Tensor:
hidden = []
for _prefill_idx in range(getattr(attn_metadata, "num_prefills", 0)):
if _prefill_idx >= len(attn_metadata.query_start_loc):
break
if _prefill_idx >= len(state_indices_tensor):
break
offset = attn_metadata.num_decode_tokens
_start = attn_metadata.query_start_loc[offset + _prefill_idx]
_end = attn_metadata.query_start_loc[offset + _prefill_idx + 1]
slot_id = state_indices_tensor[offset + _prefill_idx]
qs = q[_start:_end].transpose(0, 1).contiguous()
ks = k[_start:_end].transpose(0, 1).contiguous()
vs = v[_start:_end].transpose(0, 1).contiguous()
slice_layer_cache = kv_cache[slot_id, ...]
out_slice = prefix_fn(
qs,
ks,
vs,
slice_layer_cache,
slope_rate,
block_size,
layer_idx=layer_idx,
)
hidden.append(out_slice.contiguous())
if attn_metadata.num_decode_tokens > 0:
hidden_decode = decode_fn(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
hidden.insert(0, hidden_decode)
if not hidden:
return torch.empty((0, q.size(-1)), device=q.device, dtype=q.dtype)
hidden = torch.concat(hidden, dim=0).contiguous()
return hidden
class MiniMaxText01LinearKernel:
@staticmethod
def jit_linear_forward_prefix(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_caches: torch.Tensor,
slope_rate: torch.Tensor,
block_size: int,
layer_idx: int | None = None,
**kwargs,
) -> torch.Tensor:
slope_rate = slope_rate.to(torch.float32)
should_pad_dim = q.dim() == 3
if should_pad_dim:
q = q.unsqueeze(0)
k = k.unsqueeze(0)
v = v.unsqueeze(0)
b, h, n, d = q.shape
e = d
kv_history = kv_caches.reshape(1, h, d, e).contiguous()
output, kv_history = lightning_attention(
q, k, v, slope_rate, block_size=block_size, kv_history=kv_history
)
kv_caches.copy_(kv_history[:, :, -1, :, :].reshape(h, d, e))
assert output.shape[0] == 1, "batch size must be 1"
return rearrange(output.squeeze(0), "h n d -> n (h d)")
@PluggableLayer.register("minimax_text_01_attention")
class MiniMaxText01LinearAttention(LinearAttention):
def __init__(
self,
config,
vllm_config,
prefix: str = "linear_attn",
) -> None:
super().__init__(config, vllm_config, prefix)
self.tp_heads = self.num_heads // self.tp_size
self.qkv_size = self.num_heads * self.head_dim
self.tp_hidden = self.head_dim * self.tp_heads
self.qkv_proj = ColumnParallelLinear(
self.hidden_size,
self.hidden_inner_size * 3,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.output_gate = ColumnParallelLinear(
self.hidden_size,
self.hidden_inner_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.output_gate",
)
self.out_proj = RowParallelLinear(
self.hidden_inner_size,
self.hidden_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.out_proj",
)
self.norm = MiniMaxText01RMSNormTP(
self.hidden_inner_size,
eps=1e-5,
)
slope_rate = MiniMaxText01LinearAttention._build_slope_tensor(self.num_heads)
if self.num_hidden_layers <= 1:
self.slope_rate = slope_rate * (1 + 1e-5)
else:
self.slope_rate = slope_rate * (
1 - self.layer_idx / (self.num_hidden_layers - 1) + 1e-5
)
self.tp_slope = self.slope_rate[
self.tp_rank * self.tp_heads : (self.tp_rank + 1) * self.tp_heads
].contiguous()
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
@staticmethod
def weight_direct_load(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
assert param.size() == loaded_weight.size()
param.data.copy_(loaded_weight)
return
@staticmethod
def _build_slope_tensor(n_attention_heads: int):
def get_slopes(n):
def get_slopes_power_of_2(n):
start = 2 ** (-(2 ** -(math.log2(n) - 3)))
ratio = start
return [start * ratio**i for i in range(n)]
if math.log2(n).is_integer():
return get_slopes_power_of_2(n)
else:
closest_power_of_2 = 2 ** math.floor(math.log2(n))
return (
get_slopes_power_of_2(closest_power_of_2)
+ get_slopes(2 * closest_power_of_2)[0::2][: n - closest_power_of_2]
)
slopes = torch.tensor(
get_slopes(n_attention_heads), dtype=torch.float32
).reshape(n_attention_heads, 1, 1)
return slopes
def _prefill_and_mix_infer(
self, q, k, v, kv_cache, state_indices_tensor, attn_metadata
):
return linear_attention_prefill_and_mix(
q=q,
k=k,
v=v,
kv_cache=kv_cache,
state_indices_tensor=state_indices_tensor,
attn_metadata=attn_metadata,
slope_rate=self.tp_slope,
block_size=self.BLOCK,
decode_fn=self._decode_infer,
prefix_fn=MiniMaxText01LinearKernel.jit_linear_forward_prefix,
layer_idx=self.layer_idx,
)
def _decode_infer(self, q, k, v, kv_cache, state_indices_tensor, attn_metadata):
hidden = linear_attention_decode(
q,
k,
v,
kv_cache,
self.tp_slope,
state_indices_tensor,
q_start=0,
q_end=attn_metadata.num_decode_tokens,
slot_start=0,
slot_end=attn_metadata.num_decodes,
block_size=32,
)
return hidden
def forward(
self, hidden_states: torch.Tensor, output: torch.Tensor, positions: torch.Tensor
) -> None:
torch.ops.vllm.linear_attention(
hidden_states,
output,
positions,
self.prefix,
)
def _forward(
self, hidden_states: torch.Tensor, output: torch.Tensor, positions: torch.Tensor
) -> None:
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, LinearAttentionMetadata)
num_actual_tokens = (
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
)
else:
num_actual_tokens = hidden_states.shape[0]
qkv, _ = self.qkv_proj(hidden_states[:num_actual_tokens])
qkv32 = qkv.to(torch.float32)
qkvact = torch.nn.functional.silu(qkv32)
qkvact = qkvact.view((qkv.shape[0], self.tp_heads, -1))
q, k, v = torch.split(qkvact, [self.head_dim] * 3, dim=-1)
if attn_metadata is not None:
kv_cache = self.kv_cache[0]
state_indices_tensor = attn_metadata.state_indices_tensor
clear_linear_attention_cache_for_new_sequences(
kv_cache, state_indices_tensor, attn_metadata
)
decode_only = getattr(attn_metadata, "num_prefills", 0) == 0
if attn_metadata is None:
hidden = torch.empty(
(q.shape[0], q.shape[1] * q.shape[2]), device=q.device, dtype=q.dtype
)
else:
if not decode_only:
hidden = self._prefill_and_mix_infer(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
else:
hidden = self._decode_infer(
q, k, v, kv_cache, state_indices_tensor, attn_metadata
)
hidden = self.norm._forward(hidden)
gate, _ = self.output_gate(hidden_states[:num_actual_tokens])
hidden = F.sigmoid(gate) * hidden
hidden = hidden.to(hidden_states.dtype)
output[:num_actual_tokens], _ = self.out_proj(hidden)
@eager_break_during_capture
def linear_attention(
hidden_states: torch.Tensor,
output: torch.Tensor,
positions: torch.Tensor,
layer_name: str,
) -> None:
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._forward(hidden_states=hidden_states, output=output, positions=positions)
direct_register_custom_op(
op_name="linear_attention",
op_func=linear_attention,
mutates_args=["output"],
)
@@ -1,485 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import NamedTuple
import torch
from torch import nn
from torch.nn.parameter import Parameter
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.mamba.abstract import MambaBase
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateDtypeCalculator,
MambaStateShapeCalculator,
is_conv_state_dim_first,
)
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_scan_fn
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import selective_state_update
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
# --8<-- [start:mamba_mixer]
@PluggableLayer.register("mamba_mixer")
class MambaMixer(MambaBase, PluggableLayer):
"""Compute ∆, A, B, C, and D the state space parameters and compute
the `contextualized_states`. A, D are input independent
(see Mamba paper [1] Section 3.5.2 "Interpretation of A"
for why A isn't selective) ∆, B, C are input-dependent
(this is a key difference between Mamba and the linear time
invariant S4, and is why Mamba is called
**selective** state spaces)
"""
# --8<-- [end:mamba_mixer]
def __init__(
self,
hidden_size: int,
ssm_state_size: int,
conv_kernel_size: int,
intermediate_size: int,
time_step_rank: int,
use_conv_bias: bool,
use_bias: bool,
use_rms_norm: bool,
rms_norm_has_weight: bool = True,
rms_norm_eps: float = 1e-5,
activation="silu",
is_lora_enabled: bool = False,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
):
super().__init__()
self.time_step_rank = time_step_rank
self.ssm_state_size = ssm_state_size
self.use_rms_norm = use_rms_norm
self.activation = activation
self.is_lora_enabled = is_lora_enabled
self.conv_kernel_size = conv_kernel_size
self.intermediate_size = intermediate_size
self.conv1d = ColumnParallelLinear(
input_size=conv_kernel_size,
output_size=intermediate_size,
bias=use_conv_bias,
prefix=f"{prefix}.conv1d",
)
# unsqueeze to fit conv1d weights shape into the linear weights shape.
# Can't do this in `weight_loader` since it already exists in
# `ColumnParallelLinear` and `set_weight_attrs`
# doesn't allow to override it
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
self.in_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=use_bias,
prefix=f"{prefix}.in_proj",
)
# selective projection used to make dt, B and C input dependent
self.x_proj = RowParallelLinear(
intermediate_size,
time_step_rank + ssm_state_size * 2,
bias=False,
prefix=f"{prefix}.x_proj",
)
# time step projection (discretization) -
# In the forward we need to apply dt_proj without the bias,
# as the bias is added in the selective scan kernel.
self.dt_proj = ColumnParallelLinear(
time_step_rank,
intermediate_size,
bias=True,
skip_bias_add=True,
prefix=f"{prefix}.dt_proj",
)
def weight_loader(param: Parameter, loaded_weight: torch.Tensor):
tp_rank = get_tensor_model_parallel_rank()
tp_size = get_tensor_model_parallel_world_size()
param.data.copy_(
loaded_weight.data.split(loaded_weight.shape[0] // tp_size, dim=0)[
tp_rank
]
)
def A_weight_loader(param: Parameter, loaded_weight: torch.Tensor):
weight_loader(param, -torch.exp(loaded_weight.float()))
tp_size = get_tensor_model_parallel_world_size()
self.A = nn.Parameter(
torch.empty(
intermediate_size // tp_size,
ssm_state_size,
dtype=torch.float32,
)
)
self.D = nn.Parameter(torch.ones(intermediate_size // tp_size))
set_weight_attrs(self.D, {"weight_loader": weight_loader})
set_weight_attrs(self.A, {"weight_loader": A_weight_loader})
self.out_proj = RowParallelLinear(
intermediate_size,
hidden_size,
bias=use_bias,
input_is_parallel=True,
prefix=f"{prefix}.out_proj",
)
self.dt_layernorm = (
RMSNorm(
time_step_rank,
eps=rms_norm_eps,
has_weight=rms_norm_has_weight,
)
if use_rms_norm
else None
)
self.b_layernorm = (
RMSNorm(
ssm_state_size,
eps=rms_norm_eps,
has_weight=rms_norm_has_weight,
)
if use_rms_norm
else None
)
self.c_layernorm = (
RMSNorm(
ssm_state_size,
eps=rms_norm_eps,
has_weight=rms_norm_has_weight,
)
if use_rms_norm
else None
)
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
# The inner tuple is (conv_state, ssm_state)
self.kv_cache = (torch.tensor([]), torch.tensor([]))
self.model_config = model_config
self.cache_config = cache_config
self.prefix = prefix
def _ssm_transform(
self, x: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
# LoRA kernel requires contiguous tensor.
# ROCm: Non-contiguous tensors cause incorrect GEMM
# results when batch > 1.
if self.is_lora_enabled or current_platform.is_rocm():
x = x.contiguous()
ssm_params = self.x_proj(x)[0]
time_step, B, C = torch.split(
ssm_params,
[self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
dim=-1,
)
if self.use_rms_norm:
assert self.dt_layernorm is not None
assert self.b_layernorm is not None
assert self.c_layernorm is not None
time_step = self.dt_layernorm(time_step.contiguous())
B = self.b_layernorm(B.contiguous())
C = self.c_layernorm(C.contiguous())
# ROCm: tensor from split is non-contiguous, causing incorrect
# GEMM results in dt_proj.
if current_platform.is_rocm():
time_step = time_step.contiguous()
discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
return discrete_time_step, B, C
def forward(self, hidden_states: torch.Tensor, output: torch.Tensor):
torch.ops.vllm.mamba_mixer(
hidden_states,
output,
_encode_layer_name(self.prefix),
)
def forward_impl(self, hidden_states: torch.Tensor, output: torch.Tensor):
"""Run the Mamba-1 SSM pipeline.
Steps
-----
1. Apply the gated-MLP linear projection to the raw input.
2. Pass the projected sequence through the convolutional mixing layer.
3. Feed the result into the State-Space Model (SSM) blocks.
4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
to produce contextual representations.
5. Project the contextualised sequence back
to the output embedding dimension.
Batch handling
--------------
Prefill and decode tokens are processed by dedicated CUDA
kernels for both the convolutional (conv1d) and SSM stages.
In the case of a mixed batch (containing both prefill and
decode tokens), both sets of kernels are executed independently
and their outputs are concatenated before the final output projection.
"""
forward_context: ForwardContext = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
assert self.cache_config is not None
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, Mamba1AttentionMetadata)
query_start_loc_p = attn_metadata.query_start_loc_p
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
conv_state = (
self.kv_cache[0]
if is_conv_state_dim_first()
else self.kv_cache[0].transpose(-1, -2)
)
ssm_state = self.kv_cache[1]
has_initial_states_p = attn_metadata.has_initial_states_p
cu_chunk_seqlen_p = attn_metadata.cu_chunk_seqlen_p
last_chunk_indices_p = attn_metadata.last_chunk_indices_p
# 1. Gated MLP's linear projection
projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
hidden_states_BC, gate = projected_states.chunk(2, dim=-2)
conv_weights = self.conv1d.weight.view(
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
)
if attn_metadata is None:
# V1 profile run
hidden_states_BC = hidden_states_BC.contiguous()
return self.out_proj(hidden_states_BC.transpose(-2, -1))[0]
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
num_decode_tokens = attn_metadata.num_decode_tokens
has_prefill = num_prefill_tokens > 0
has_decode = num_decode_tokens > 0
num_actual_tokens = num_prefill_tokens + num_decode_tokens
prefill_decode_split = split_batch_to_prefill_and_decode(
hidden_states_BC,
gate,
num_prefill_tokens,
num_decode_tokens,
)
hidden_states_BC_p = prefill_decode_split.hidden_states_BC_p
hidden_states_BC_d = prefill_decode_split.hidden_states_BC_d
gate_p = prefill_decode_split.gate_p
gate_d = prefill_decode_split.gate_d
ssm_outputs = []
if has_prefill:
# 2. Convolution sequence transformation
conv_out_p = causal_conv1d_fn(
hidden_states_BC_p,
conv_weights,
self.conv1d.bias,
activation=self.activation,
conv_states=conv_state,
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
query_start_loc=query_start_loc_p,
metadata=attn_metadata,
)
# 3. State Space Model sequence transformations.
discrete_time_step_p, B_p, C_p = self._ssm_transform(
conv_out_p.transpose(-2, -1)
)
time_proj_bias = self._time_proj_bias()
# 4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
scan_out_p = selective_scan_fn(
conv_out_p,
ssm_state,
discrete_time_step_p,
self.A,
B_p.transpose(-2, -1),
C_p.transpose(-2, -1),
self.D.float(),
gate_p,
time_proj_bias,
delta_softplus=True,
cache_indices=state_indices_tensor_p,
has_initial_state=has_initial_states_p,
query_start_loc=query_start_loc_p,
cu_chunk_seqlen=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
)
ssm_outputs.append(scan_out_p)
if has_decode:
# state_indices_tensor_d is assigned when attn_metadata is not None,
# and has_decode is only True when attn_metadata is not None
assert state_indices_tensor_d is not None
# 2. Convolution sequence transformation
conv_out_d = causal_conv1d_update(
hidden_states_BC_d.transpose(0, 1),
conv_state,
conv_weights,
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices_tensor_d,
).transpose(0, 1)
# 3. State Space Model sequence transformation.
discrete_time_step_d, B_d, C_d = self._ssm_transform(
conv_out_d.transpose(-2, -1)
)
time_proj_bias = self._time_proj_bias()
# 4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
scan_outputs_d = torch.empty_like(hidden_states_BC_d.transpose(0, 1))
selective_state_update(
ssm_state,
conv_out_d.transpose(0, 1),
discrete_time_step_d.transpose(0, 1),
self.A,
B_d,
C_d,
self.D,
time_proj_bias,
z=gate_d.transpose(0, 1),
dt_softplus=True,
state_batch_indices=state_indices_tensor_d,
dst_state_batch_indices=state_indices_tensor_d,
out=scan_outputs_d,
)
scan_outputs_d = scan_outputs_d.transpose(0, 1)
ssm_outputs.insert(0, scan_outputs_d)
scan_outputs_combined = (
ssm_outputs[0] if len(ssm_outputs) == 1 else torch.cat(ssm_outputs, dim=-1)
)
# 5. Final output projection
if self.is_lora_enabled: # Lora kernel requires contiguous tensor.
scan_outputs_combined = scan_outputs_combined.transpose(-2, -1).contiguous()
out = self.out_proj(scan_outputs_combined)[0]
else:
out = self.out_proj(scan_outputs_combined.transpose(-2, -1))[0]
output[:num_actual_tokens] = out
def get_state_dtype(self) -> tuple[torch.dtype]:
assert self.model_config is not None
assert self.cache_config is not None
return MambaStateDtypeCalculator.mamba1_state_dtype(
self.model_config.dtype,
self.cache_config.mamba_cache_dtype,
self.cache_config.mamba_ssm_cache_dtype,
)
def get_state_shape(self) -> tuple[tuple[int, ...], tuple[int, ...]]:
return MambaStateShapeCalculator.mamba1_state_shape(
tp_world_size=get_tensor_model_parallel_world_size(),
intermediate_size=self.intermediate_size,
state_size=self.ssm_state_size,
conv_kernel=self.conv_kernel_size,
)
@property
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.MAMBA1
def _time_proj_bias(self) -> torch.Tensor | None:
if hasattr(self.dt_proj, "bias") and self.dt_proj.bias is not None:
return self.dt_proj.bias.float()
return None
class PrefillDecodeSplit(NamedTuple):
hidden_states_BC_p: torch.Tensor
hidden_states_BC_d: torch.Tensor
gate_p: torch.Tensor
gate_d: torch.Tensor
def split_batch_to_prefill_and_decode(
hidden_states_BC: torch.Tensor,
gate: torch.Tensor,
num_prefill_tokens: int,
num_decode_tokens: int,
) -> PrefillDecodeSplit:
num_actual_tokens = num_prefill_tokens + num_decode_tokens
# In v1, decode tokens come first, then prefill tokens.
hidden_states_BC_d, hidden_states_BC_p = torch.split(
hidden_states_BC[..., :num_actual_tokens],
[num_decode_tokens, num_prefill_tokens],
dim=-1,
)
gate_d, gate_p = torch.split(
gate[..., :num_actual_tokens], [num_decode_tokens, num_prefill_tokens], dim=-1
)
return PrefillDecodeSplit(
hidden_states_BC_p=hidden_states_BC_p,
hidden_states_BC_d=hidden_states_BC_d,
gate_p=gate_p,
gate_d=gate_d,
)
@eager_break_during_capture
def mamba_mixer(
hidden_states: torch.Tensor,
output: torch.Tensor,
layer_name: LayerNameType,
) -> None:
layer_name = _resolve_layer_name(layer_name)
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self.forward_impl(hidden_states=hidden_states, output=output)
direct_register_custom_op(
op_name="mamba_mixer",
op_func=mamba_mixer,
mutates_args=["output"],
)
@@ -1,83 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@triton.jit
def _gather_initial_states_kernel(
state_ptr,
indices_ptr,
has_initial_state_ptr,
output_ptr,
stride_state_batch,
stride_indices,
stride_has_initial_state,
row_size: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
):
block_idx = tl.program_id(0)
batch_idx = tl.program_id(1)
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < row_size
if launch_pdl:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
has_initial_state = tl.load(
has_initial_state_ptr + batch_idx * stride_has_initial_state
).to(tl.int1)
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
state_idx = tl.where(has_initial_state, state_idx, 0)
values = tl.load(
state_ptr + state_idx * stride_state_batch + offsets,
mask=mask & has_initial_state,
other=0.0,
)
tl.store(output_ptr + batch_idx * row_size + offsets, values, mask=mask)
def gather_initial_states(
state: torch.Tensor,
indices: torch.Tensor,
has_initial_state: torch.Tensor,
) -> torch.Tensor:
"""Gather dense state rows, replacing uninitialized rows with zeros."""
assert state.ndim >= 2
assert state.is_cuda or state.is_xpu
assert indices.ndim == 1 and has_initial_state.ndim == 1
assert indices.shape == has_initial_state.shape
assert indices.device == state.device
assert has_initial_state.device == state.device
assert indices.dtype in (torch.int32, torch.int64)
assert has_initial_state.dtype == torch.bool
row_size = state[0].numel()
# Mamba pages may pad stride(0), but each state row remains dense.
assert state[0].is_contiguous()
output = torch.empty(
(indices.numel(), *state.shape[1:]),
dtype=state.dtype,
device=state.device,
)
block_size = min(triton.next_power_of_2(row_size), 1024)
grid = (triton.cdiv(row_size, block_size), indices.numel())
_gather_initial_states_kernel[grid](
state,
indices,
has_initial_state,
output,
state.stride(0),
indices.stride(0),
has_initial_state.stride(0),
row_size=row_size,
BLOCK_SIZE=block_size,
num_warps=8,
launch_pdl=current_platform.is_arch_support_pdl(),
)
return output
@@ -1,105 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup import WarmupIntRange
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
DispatchSpec,
TritonWarmupTensor,
triton_kernel_dispatcher_with_warmup,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@triton.jit
def _scatter_states_kernel(
state_ptr,
src_ptr,
indices_ptr,
stride_state_batch,
stride_src_batch,
stride_indices,
row_size: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
):
block_idx = tl.program_id(0)
batch_idx = tl.program_id(1)
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < row_size
if launch_pdl:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
values = tl.load(src_ptr + batch_idx * stride_src_batch + offsets, mask=mask)
tl.store(state_ptr + state_idx * stride_state_batch + offsets, values, mask=mask)
def _scatter_states_warmup_inputs(
*,
state_shape: tuple[int, ...],
state_dtype: torch.dtype,
indices_dtype: torch.dtype,
max_num_tokens: int,
) -> dict[str, object]:
num_tokens: Any = WarmupIntRange(1, max_num_tokens + 1)
return dict(
state=TritonWarmupTensor(
state_dtype,
shape=(1,) + state_shape,
),
src=TritonWarmupTensor(
state_dtype,
shape=(num_tokens,) + state_shape,
),
indices=TritonWarmupTensor(
indices_dtype,
shape=(num_tokens,),
),
)
@triton_kernel_dispatcher_with_warmup(
kernel=_scatter_states_kernel,
warmup_inputs=_scatter_states_warmup_inputs,
)
def scatter_states(
state: torch.Tensor,
src: torch.Tensor,
indices: torch.Tensor,
) -> DispatchSpec:
"""Scatter ``src`` rows into ``state`` at ``indices`` (in place).
Equivalent to ``state[indices] = src`` but non-atomic and bandwidth-bound,
since mamba cache slots are unique per sequence. ``gather_initial_states``
is the read-side counterpart.
"""
assert state.ndim >= 2
assert state.is_cuda
assert src.ndim == state.ndim
assert indices.ndim == 1
assert indices.device == state.device
assert src.shape[1:] == state.shape[1:]
assert src.shape[0] == indices.shape[0]
assert indices.dtype in (torch.int32, torch.int64)
row_size = state[0].numel()
assert state[0].is_contiguous()
assert src[0].is_contiguous()
block_size = min(triton.next_power_of_2(row_size), 1024)
grid = (triton.cdiv(row_size, block_size), indices.numel())
return grid, dict(
stride_state_batch=state.stride(0),
stride_src_batch=src.stride(0),
stride_indices=indices.stride(0),
row_size=row_size,
BLOCK_SIZE=block_size,
num_warps=8,
launch_pdl=current_platform.is_arch_support_pdl(),
)
File diff suppressed because it is too large Load Diff
@@ -1,5 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .abstract import *
from .common import *
from .special import *
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Set
import torch
import torch.nn as nn
from vllm.tasks import PoolingTask
from vllm.v1.outputs import PoolerOutput
from vllm.v1.pool.metadata import PoolingMetadata
from .common import PoolingParamsUpdate
class Pooler(nn.Module, ABC):
"""The interface required for all poolers used in pooling models in vLLM."""
@abstractmethod
def get_supported_tasks(self) -> Set[PoolingTask]:
"""Determine which pooling tasks are supported."""
raise NotImplementedError
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
"""Construct the updated pooling parameters to use for a supported task."""
return PoolingParamsUpdate()
@abstractmethod
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> PoolerOutput:
raise NotImplementedError
__all__ = ["Pooler"]
@@ -1,158 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Callable
from typing import TypeVar
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import PreTrainedConfig
from vllm.config import ModelConfig
from vllm.logger import init_logger
from vllm.utils.import_utils import resolve_obj_by_qualname
logger = init_logger(__name__)
def get_act_fn(
config: PreTrainedConfig,
static_num_labels: bool = True,
) -> "PoolerActivation":
# get classification act_fn
# Implement alignment with transformers ForSequenceClassificationLoss
# https://github.com/huggingface/transformers/blob/57bb6db6ee4cfaccc45b8d474dfad5a17811ca60/src/transformers/loss/loss_utils.py#L92
num_labels: int | None = None
if static_num_labels:
num_labels = getattr(config, "num_labels", 0)
problem_type = getattr(config, "problem_type", "")
if problem_type == "regression":
return PoolerIdentity()
if problem_type == "single_label_classification":
return PoolerClassify(num_labels=num_labels)
if problem_type == "multi_label_classification":
return PoolerMultiLabelClassify()
# get cross_encoder act_fn
function_name: str | None = None
if (
hasattr(config, "sentence_transformers")
and "activation_fn" in config.sentence_transformers
):
function_name = config.sentence_transformers["activation_fn"]
elif (
hasattr(config, "sbert_ce_default_activation_function")
and config.sbert_ce_default_activation_function is not None
):
function_name = config.sbert_ce_default_activation_function
if function_name is not None:
if not function_name.startswith("torch.nn.modules."):
raise ValueError(
"Loading of activation functions is restricted to "
"torch.nn.modules for security reasons"
)
fn = resolve_obj_by_qualname(function_name)()
return PoolerActivation.wraps(fn)
return PoolerClassify(num_labels=num_labels)
def resolve_classifier_act_fn(
model_config: ModelConfig,
static_num_labels: bool = True,
act_fn: "PoolerActivation | None" = None,
) -> "PoolerActivation":
if act_fn is None:
return get_act_fn(model_config.hf_config, static_num_labels)
if not callable(act_fn):
raise TypeError(f"Expected a callable activation function, got {type(act_fn)}")
return act_fn
_T = TypeVar("_T", torch.Tensor, list[torch.Tensor])
class PoolerActivation(nn.Module, ABC):
@staticmethod
def wraps(module: nn.Module) -> "PoolerActivation":
if isinstance(module, nn.Identity):
return PoolerIdentity()
if isinstance(module, (nn.Sigmoid, nn.Softmax)):
return PoolerClassify()
return LambdaPoolerActivation(module)
@abstractmethod
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
raise NotImplementedError
def forward(self, pooled_data: _T) -> _T:
# shape:
# classify -> (batch_size, num_classes)
# embed -> (batch_size, embedding_size) or list(embedding_size)
if isinstance(pooled_data, list):
return [self.forward_chunk(data) for data in pooled_data]
return self.forward_chunk(pooled_data)
class PoolerIdentity(PoolerActivation):
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
return pooled_data
class PoolerNormalize(PoolerActivation):
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
return F.normalize(pooled_data, p=2, dim=-1)
class PoolerMultiLabelClassify(PoolerActivation):
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
return F.sigmoid(pooled_data)
class PoolerClassify(PoolerActivation):
def __init__(self, *, num_labels: int | None = None) -> None:
super().__init__()
if num_labels == 0:
logger.warning(
"num_labels should be > 0 for classification "
"models, falling back to sigmoid. "
"Please check if the configuration is correct."
)
self.num_labels = num_labels
def extra_repr(self) -> str:
return f"num_labels={self.num_labels}"
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
num_labels = self.num_labels
if num_labels is None:
num_labels = pooled_data.shape[-1]
if num_labels < 2:
return F.sigmoid(pooled_data)
return F.softmax(pooled_data, dim=-1)
class LambdaPoolerActivation(PoolerActivation):
def __init__(self, fn: Callable[[torch.Tensor], torch.Tensor]) -> None:
super().__init__()
self.fn = fn
def extra_repr(self) -> str:
name = getattr(self.fn, "__name__", None)
if name is None:
name = self.fn.__class__.__name__
return f"fn={name}"
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
return self.fn(pooled_data)
@@ -1,32 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from dataclasses import dataclass
from typing import TypeVar
import torch
from vllm.pooling_params import PoolingParams
_T = TypeVar("_T", bound=torch.Tensor | list[torch.Tensor])
ProjectorFn = Callable[[torch.Tensor], torch.Tensor]
ClassifierFn = Callable[[torch.Tensor], torch.Tensor]
ActivationFn = Callable[[_T], _T]
@dataclass(frozen=True)
class PoolingParamsUpdate:
requires_token_ids: bool = False
"""Set this flag to enable CPU prompt token IDs for your pooler."""
def __or__(self, other: "PoolingParamsUpdate") -> "PoolingParamsUpdate":
return PoolingParamsUpdate(
requires_token_ids=self.requires_token_ids or other.requires_token_ids,
)
def apply(self, params: PoolingParams) -> None:
params.requires_token_ids = self.requires_token_ids
__all__ = ["ActivationFn", "ClassifierFn", "ProjectorFn", "PoolingParamsUpdate"]
@@ -1,45 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Poolers that produce an output aggregating all tokens in the sequence."""
from .heads import (
ClassifierPoolerHead,
EmbeddingPoolerHead,
SequencePoolerHead,
SequencePoolerHeadOutput,
)
from .methods import (
CLSPool,
LastPool,
MeanPool,
SequencePoolingMethod,
SequencePoolingMethodOutput,
get_seq_pooling_method,
)
from .poolers import (
SequencePooler,
SequencePoolerOutput,
SequencePoolingFn,
SequencePoolingHeadFn,
pooler_for_classify,
pooler_for_embed,
)
__all__ = [
"SequencePoolerHead",
"SequencePoolerHeadOutput",
"ClassifierPoolerHead",
"EmbeddingPoolerHead",
"SequencePoolingMethod",
"SequencePoolingMethodOutput",
"CLSPool",
"LastPool",
"MeanPool",
"get_seq_pooling_method",
"SequencePooler",
"SequencePoolingFn",
"SequencePoolingHeadFn",
"SequencePoolerOutput",
"pooler_for_classify",
"pooler_for_embed",
]
@@ -1,204 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Set
from typing import TypeAlias
import torch
import torch.nn as nn
from vllm.model_executor.layers.pooler import ActivationFn, ClassifierFn, ProjectorFn
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata
from .methods import SequencePoolingMethodOutput
SequencePoolerHeadOutput: TypeAlias = torch.Tensor | list[torch.Tensor]
class SequencePoolerHead(nn.Module, ABC):
@abstractmethod
def get_supported_tasks(self) -> Set[PoolingTask]:
raise NotImplementedError
@abstractmethod
def forward(
self,
pooled_data: SequencePoolingMethodOutput,
pooling_metadata: PoolingMetadata,
) -> SequencePoolerHeadOutput:
raise NotImplementedError
class EmbeddingPoolerHead(SequencePoolerHead):
def __init__(
self,
projector: ProjectorFn | None = None,
head_dtype: torch.dtype | str | None = None,
activation: ActivationFn | None = None,
) -> None:
super().__init__()
self.projector = projector
self.head_dtype = head_dtype
self.activation = activation
def extra_repr(self) -> str:
attrs = []
if self.head_dtype is not None:
attrs.append(f"head_dtype={self.head_dtype}")
if self.projector is not None:
attrs.append("projector=True")
if self.activation is not None:
attrs.append(f"activation={self.activation.__class__.__name__}")
return ", ".join(attrs)
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"embed"}
def forward(
self,
pooled_data: SequencePoolingMethodOutput,
pooling_metadata: PoolingMetadata,
) -> SequencePoolerHeadOutput:
pooling_params = pooling_metadata.pooling_params
if len(pooled_data) != len(pooling_params):
raise ValueError(
f"pooled_data length ({len(pooled_data)}) does not match "
f"pooling_params length ({len(pooling_params)})"
)
if isinstance(pooled_data, list):
pooled_data = torch.stack(pooled_data)
# pooled_data shape: [batchsize, hidden_size]
if self.head_dtype is not None:
pooled_data = pooled_data.to(self.head_dtype)
# Apply ST projector
if self.projector is not None:
embeddings = self.projector(pooled_data)
else:
embeddings = pooled_data
# embeddings shape: [batchsize, embedding_size]
# for matryoshka representation
dimensions_list = [pooling_param.dimensions for pooling_param in pooling_params]
if any(d is not None for d in dimensions_list):
# change the output dimension
if len(embeddings) != len(dimensions_list):
raise ValueError(
f"embeddings length ({len(embeddings)}) does not match "
f"dimensions_list length ({len(dimensions_list)})"
)
if len(set(dimensions_list)) == 1 and not isinstance(embeddings, list):
# if all dimensions are the same
d = dimensions_list[0]
embeddings = embeddings[..., :d]
else:
embeddings = [
vecs if d is None else vecs[..., :d]
for vecs, d in zip(embeddings, dimensions_list)
]
# for normalize
if self.activation is not None:
flags = [p.use_activation for p in pooling_params]
if len(set(flags)) == 1:
if flags[0]:
embeddings = self.activation(embeddings)
else:
embeddings = [
self.activation(vecs) if f else vecs
for vecs, f in zip(embeddings, flags)
]
# embeddings shape: [batchsize, embedding_size]
return embeddings
class ClassifierPoolerHead(SequencePoolerHead):
def __init__(
self,
classifier: ClassifierFn | None = None,
logit_mean: float | None = None,
logit_sigma: float | None = None,
head_dtype: torch.dtype | str | None = None,
activation: ActivationFn | None = None,
) -> None:
super().__init__()
self.classifier = classifier
self.logit_mean = logit_mean
self.logit_sigma = logit_sigma
self.head_dtype = head_dtype
self.activation = activation
def extra_repr(self) -> str:
attrs = []
if self.head_dtype is not None:
attrs.append(f"head_dtype={self.head_dtype}")
if self.classifier is not None:
attrs.append("classifier=True")
if self.logit_mean is not None:
attrs.append(f"logit_mean={self.logit_mean}")
if self.logit_sigma is not None:
attrs.append(f"logit_sigma={self.logit_sigma}")
if self.activation is not None:
attrs.append(f"activation={self.activation.__class__.__name__}")
return ", ".join(attrs)
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"classify"}
def forward(
self,
pooled_data: SequencePoolingMethodOutput,
pooling_metadata: PoolingMetadata,
) -> SequencePoolerHeadOutput:
pooling_params = pooling_metadata.pooling_params
if len(pooled_data) != len(pooling_params):
raise ValueError(
f"pooled_data length ({len(pooled_data)}) does not match "
f"pooling_params length ({len(pooling_params)})"
)
if isinstance(pooled_data, list):
pooled_data = torch.stack(pooled_data)
# pooled_data shape: [batchsize, hidden_size]
if self.head_dtype is not None:
pooled_data = pooled_data.to(self.head_dtype)
if self.classifier is not None:
logits: SequencePoolerHeadOutput = self.classifier(pooled_data)
else:
logits = pooled_data
# logits shape: [batchsize, num_labels]
# Affine score calibration: activation((logit - mean) / sigma)
if self.logit_mean is not None:
logits = (
[value - self.logit_mean for value in logits]
if isinstance(logits, list)
else logits - self.logit_mean
)
if self.logit_sigma is not None:
logits = (
[value / self.logit_sigma for value in logits]
if isinstance(logits, list)
else logits / self.logit_sigma
)
if self.activation is not None:
flags = [p.use_activation for p in pooling_params]
if len(set(flags)) == 1:
logits = self.activation(logits) if flags[0] else logits
else:
logits = [
self.activation(vecs) if f else vecs
for vecs, f in zip(logits, flags)
]
# logits shape: [batchsize, num_labels]
return logits
@@ -1,119 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Set
from typing import TypeAlias
import torch
import torch.nn as nn
from vllm.config.pooler import SequencePoolingType
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.tasks import PoolingTask
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.pool.metadata import PoolingMetadata
SequencePoolingMethodOutput: TypeAlias = torch.Tensor | list[torch.Tensor]
_MEAN_POOL_ACCUMULATION_CHUNK_BYTES = 16 * 1024 * 1024 # 16MB
class SequencePoolingMethod(nn.Module, ABC):
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"token_embed", "token_classify", "embed", "classify"}
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return PoolingParamsUpdate()
@abstractmethod
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> SequencePoolingMethodOutput:
raise NotImplementedError
class CLSPool(SequencePoolingMethod):
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> SequencePoolingMethodOutput:
pooling_cursor = pooling_metadata.get_pooling_cursor()
if pooling_cursor.is_partial_prefill():
raise RuntimeError("partial prefill is not supported with CLS pooling")
return hidden_states[pooling_cursor.first_token_indices_gpu]
class LastPool(SequencePoolingMethod):
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> SequencePoolingMethodOutput:
pooling_cursor = pooling_metadata.get_pooling_cursor()
return hidden_states[pooling_cursor.last_token_indices_gpu]
class MeanPool(SequencePoolingMethod):
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> SequencePoolingMethodOutput:
pooling_cursor = pooling_metadata.get_pooling_cursor()
if pooling_cursor.is_partial_prefill():
raise RuntimeError("partial prefill is not supported with MEAN pooling")
prompt_lens_cpu = pooling_cursor.prompt_lens_cpu
num_seqs = prompt_lens_cpu.numel()
hidden_size = hidden_states.shape[-1]
if num_seqs == 0:
# early return for empty batch
return hidden_states.new_empty((0, hidden_size), dtype=torch.float32)
prompt_lens = async_tensor_h2d(
prompt_lens_cpu, device=hidden_states.device, dtype=torch.int64
)
# eg. [2, 1, 3] -> [0, 0, 1, 2, 2, 2]
segment_ids = torch.repeat_interleave(
torch.arange(num_seqs, device=hidden_states.device, dtype=torch.long),
prompt_lens,
output_size=int(prompt_lens_cpu.sum()),
)
segment_sums = torch.zeros(
(num_seqs, hidden_size),
dtype=torch.float32,
device=hidden_states.device,
)
bytes_per_token = hidden_size * torch.finfo(torch.float32).bits // 8
chunk_size = max(1, _MEAN_POOL_ACCUMULATION_CHUNK_BYTES // bytes_per_token)
# iterate over the batch in chunks
for start in range(0, hidden_states.shape[0], chunk_size):
end = min(start + chunk_size, hidden_states.shape[0])
# using index_add_ to accumulate for each segment
segment_sums.index_add_(
0,
segment_ids[start:end],
hidden_states[start:end].to(dtype=torch.float32),
)
return segment_sums / prompt_lens.unsqueeze(1)
def get_seq_pooling_method(
pooling_type: SequencePoolingType | str,
) -> SequencePoolingMethod:
if pooling_type == "CLS":
return CLSPool()
if pooling_type == "LAST":
return LastPool()
if pooling_type == "MEAN":
return MeanPool()
raise NotImplementedError(f"Unknown sequence pooling type: {pooling_type!r}")
@@ -1,137 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Set
from typing import TypeAlias
import torch
from vllm.config import PoolerConfig, get_current_vllm_config
from vllm.model_executor.layers.pooler import ClassifierFn, PoolingParamsUpdate
from vllm.model_executor.layers.pooler.abstract import Pooler
from vllm.model_executor.layers.pooler.activations import (
PoolerActivation,
PoolerNormalize,
resolve_classifier_act_fn,
)
from vllm.model_executor.models.adapters import _load_st_projector
from vllm.tasks import POOLING_TASKS, PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata
from .heads import (
ClassifierPoolerHead,
EmbeddingPoolerHead,
SequencePoolerHead,
SequencePoolerHeadOutput,
)
from .methods import (
SequencePoolingMethod,
SequencePoolingMethodOutput,
get_seq_pooling_method,
)
SequencePoolingFn: TypeAlias = Callable[
[torch.Tensor, PoolingMetadata],
SequencePoolingMethodOutput,
]
SequencePoolingHeadFn: TypeAlias = Callable[
[SequencePoolingMethodOutput, PoolingMetadata],
SequencePoolerHeadOutput,
]
SequencePoolerOutput: TypeAlias = torch.Tensor | list[torch.Tensor]
class SequencePooler(Pooler):
"""A layer that pools specific information from hidden states.
This layer does the following:
1. Extracts specific tokens or aggregates data based on pooling method.
2. Postprocesses the output based on pooling head.
3. Returns structured results as `PoolerOutput`.
"""
def __init__(
self,
pooling: SequencePoolingMethod | SequencePoolingFn,
head: SequencePoolerHead | SequencePoolingHeadFn,
) -> None:
super().__init__()
self.pooling = pooling
self.head = head
def extra_repr(self) -> str:
return (
f"pooling={self.pooling.__class__.__name__}, "
f"head={self.head.__class__.__name__}"
)
def get_supported_tasks(self) -> Set[PoolingTask]:
tasks = set(POOLING_TASKS)
if isinstance(self.pooling, SequencePoolingMethod):
tasks &= self.pooling.get_supported_tasks()
if isinstance(self.head, SequencePoolerHead):
tasks &= self.head.get_supported_tasks()
return tasks
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
updates = PoolingParamsUpdate()
if isinstance(self.pooling, SequencePoolingMethod):
updates |= self.pooling.get_pooling_updates(task)
return updates
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> SequencePoolerOutput:
pooled_data = self.pooling(hidden_states, pooling_metadata)
pooled_data = self.head(pooled_data, pooling_metadata)
return pooled_data
def pooler_for_embed(pooler_config: PoolerConfig) -> SequencePooler:
pooling = get_seq_pooling_method(pooler_config.get_seq_pooling_type())
vllm_config = get_current_vllm_config()
model_config = vllm_config.model_config
head = EmbeddingPoolerHead(
head_dtype=model_config.head_dtype,
projector=_load_st_projector(model_config),
activation=PoolerNormalize(),
)
return SequencePooler(pooling=pooling, head=head)
def pooler_for_classify(
pooler_config: PoolerConfig,
*,
pooling: SequencePoolingMethod | SequencePoolingFn | None = None,
classifier: ClassifierFn | None = None,
act_fn: PoolerActivation | None = None,
) -> SequencePooler:
if pooling is None:
pooling = get_seq_pooling_method(pooler_config.get_seq_pooling_type())
vllm_config = get_current_vllm_config()
model_config = vllm_config.model_config
if model_config.pooler_config is None:
raise ValueError(
"model_config.pooler_config must be set for classification pooling"
)
head = ClassifierPoolerHead(
head_dtype=model_config.head_dtype,
classifier=classifier,
logit_mean=model_config.pooler_config.logit_mean,
logit_sigma=model_config.pooler_config.logit_sigma,
activation=resolve_classifier_act_fn(
model_config, static_num_labels=False, act_fn=act_fn
),
)
return SequencePooler(pooling=pooling, head=head)
@@ -1,258 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
from collections.abc import Mapping, Set
from itertools import groupby
import torch
from vllm.config import PoolerConfig
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata
from .abstract import Pooler, PoolerOutput
from .common import ClassifierFn
from .seqwise import (
SequencePoolingFn,
SequencePoolingMethod,
pooler_for_classify,
pooler_for_embed,
)
from .tokwise import pooler_for_token_classify, pooler_for_token_embed
class DispatchPooler(Pooler):
"""Dispatches calls to a sub-pooler based on the pooling task."""
@classmethod
def for_embedding(cls, pooler_config: PoolerConfig):
return cls(
{
"token_embed": pooler_for_token_embed(pooler_config),
"embed": pooler_for_embed(pooler_config),
},
)
@classmethod
def for_seq_cls(
cls,
pooler_config: PoolerConfig,
*,
pooling: SequencePoolingMethod | SequencePoolingFn | None = None,
classifier: ClassifierFn | None = None,
):
return cls(
{
"token_classify": pooler_for_token_classify(
pooler_config,
classifier=classifier,
),
"classify": pooler_for_classify(
pooler_config,
pooling=pooling,
classifier=classifier,
),
}
)
def __init__(self, poolers_by_task: Mapping[PoolingTask, Pooler]) -> None:
super().__init__()
for task, pooler in poolers_by_task.items():
if task not in pooler.get_supported_tasks():
raise ValueError(
f"{pooler=} does not support {task=}. "
f"Supported tasks: {pooler.get_supported_tasks()}"
)
self.poolers_by_task = poolers_by_task
def get_supported_tasks(self) -> Set[PoolingTask]:
return set(self.poolers_by_task)
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return self.poolers_by_task[task].get_pooling_updates(task)
def replace_classifier(
self,
new_classifier: ClassifierFn,
) -> None:
"""Replaces the classifier to the LoRA-wrapped version."""
for task, pooler in self.poolers_by_task.items():
if task not in {"classify"}:
# Now LoRA only supports classification tasks,
# so we only replace the classifier for "classify" task.
continue
head = getattr(pooler, "head", None)
if head is not None and hasattr(head, "classifier"):
head.classifier = new_classifier
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> PoolerOutput:
poolers_by_task = self.poolers_by_task
cursor = pooling_metadata.pooling_cursor
outputs = list[torch.Tensor | None]()
offset = 0
token_offset = 0
for task, group in groupby(pooling_metadata.tasks):
if not (pooler := poolers_by_task.get(task)):
raise ValueError(
f"Unsupported task: {task!r} "
f"Supported tasks: {self.get_supported_tasks()}"
)
num_items = len(list(group))
group_metadata = pooling_metadata[offset : offset + num_items]
if cursor is None:
group_hidden_states = hidden_states
else:
# Slice out this group's tokens so sub-poolers see only their
# portion of the batch. Token offset is computed from the CPU
# `num_scheduled_tokens_cpu` to avoid a GPU->CPU sync.
group_cursor = group_metadata.pooling_cursor
assert group_cursor is not None
num_group_tokens = int(group_cursor.num_scheduled_tokens_cpu.sum())
group_hidden_states = hidden_states[
token_offset : token_offset + num_group_tokens
]
if token_offset:
# Shift first/last indices to be relative to the slice
# so seqwise poolers (which index `hidden_states` directly)
# remain correct.
pooling_cursor = dataclasses.replace(
group_cursor,
first_token_indices_gpu=(
group_cursor.first_token_indices_gpu - token_offset
),
last_token_indices_gpu=(
group_cursor.last_token_indices_gpu - token_offset
),
)
group_metadata = dataclasses.replace(
group_metadata, pooling_cursor=pooling_cursor
)
token_offset += num_group_tokens
group_output: PoolerOutput = pooler(group_hidden_states, group_metadata)
outputs.extend(group_output)
offset += num_items
return outputs
def extra_repr(self) -> str:
s = f"supported_task={self.get_supported_tasks()}"
return s
class IdentityPooler(Pooler):
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"plugin"}
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> PoolerOutput:
return hidden_states
class BOSEOSFilter(Pooler):
"""Filters the BOS and EOS token results from outputs."""
def __init__(
self,
pooler: Pooler,
bos_token_id: int = -1, # -1 disables the filtering
eos_token_id: int = -1,
) -> None:
super().__init__()
self.pooler = pooler
self.bos_token_id = bos_token_id
self.eos_token_id = eos_token_id
def extra_repr(self) -> str:
return f"bos_token_id={self.bos_token_id}, eos_token_id={self.eos_token_id}"
def get_supported_tasks(self) -> Set[PoolingTask]:
return self.pooler.get_supported_tasks()
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return PoolingParamsUpdate(requires_token_ids=True)
def forward(
self,
hidden_states: torch.Tensor | list[torch.Tensor],
pooling_metadata: PoolingMetadata,
) -> PoolerOutput:
pooled_outputs = self.pooler(hidden_states, pooling_metadata)
assert isinstance(pooled_outputs, list)
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
for i, (prompt_len, token_ids) in enumerate(
zip(pooling_metadata.prompt_lens, prompt_token_ids)
):
# Late chunking returns one row per chunk, with special tokens
# already included in the means. Skip BOS/EOS row removal for these
# requests; ordinary token outputs still use the filtering below.
if pooling_metadata.pooling_params[i].late_chunking_params is not None:
continue
pooled_data = pooled_outputs[i]
assert (
isinstance(pooled_data, torch.Tensor)
and pooled_data.shape[0] == prompt_len
)
if int(token_ids[0]) == self.bos_token_id:
pooled_data = pooled_data[1:]
if int(token_ids[-1]) == self.eos_token_id:
pooled_data = pooled_data[:-1]
pooled_outputs[i] = pooled_data.squeeze(-1)
return pooled_outputs
class BgeM3Pooler(Pooler):
def __init__(self, token_classify_pooler: Pooler, embed_pooler: Pooler) -> None:
super().__init__()
self.token_classify_pooler = token_classify_pooler
self.embed_pooler = embed_pooler
def forward(
self, hidden_states: torch.Tensor, pooling_metadata: PoolingMetadata
) -> PoolerOutput:
embed_outputs = self.embed_pooler(hidden_states, pooling_metadata)
token_classify_outputs = self.token_classify_pooler(
hidden_states, pooling_metadata
)
pooler_outputs: list[torch.Tensor] = []
for embed_output, token_classify_output in zip(
embed_outputs, token_classify_outputs
):
pooler_outputs.append(
torch.cat(
[embed_output.view(-1), token_classify_output.view(-1)], dim=-1
)
)
return pooler_outputs
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"embed&token_classify"}
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return self.embed_pooler.get_pooling_updates(
"embed"
) | self.token_classify_pooler.get_pooling_updates("token_classify")
def extra_repr(self) -> str:
s = f"supported_task={self.get_supported_tasks()}"
return s
__all__ = ["BOSEOSFilter", "DispatchPooler", "IdentityPooler", "BgeM3Pooler"]
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Poolers that produce an output for each token in the sequence."""
from .heads import (
TokenClassifierPoolerHead,
TokenEmbeddingPoolerHead,
TokenPoolerHead,
TokenPoolerHeadOutputItem,
)
from .methods import (
AllPool,
StepPool,
TokenPoolingMethod,
TokenPoolingMethodOutputItem,
get_tok_pooling_method,
)
from .poolers import (
TokenPooler,
TokenPoolerOutput,
TokenPoolingFn,
TokenPoolingHeadFn,
pooler_for_token_classify,
pooler_for_token_embed,
)
__all__ = [
"TokenPoolerHead",
"TokenPoolerHeadOutputItem",
"TokenClassifierPoolerHead",
"TokenEmbeddingPoolerHead",
"TokenPoolingMethod",
"TokenPoolingMethodOutputItem",
"AllPool",
"StepPool",
"get_tok_pooling_method",
"TokenPooler",
"TokenPoolerOutput",
"TokenPoolingFn",
"TokenPoolingHeadFn",
"pooler_for_token_classify",
"pooler_for_token_embed",
]
@@ -1,192 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Set
from typing import TypeAlias
import torch
import torch.nn as nn
from vllm.model_executor.layers.pooler import ActivationFn, ClassifierFn, ProjectorFn
from vllm.pooling_params import PoolingParams
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata
from .methods import TokenPoolingMethodOutputItem
TokenPoolerHeadOutputItem: TypeAlias = torch.Tensor | None
def _mean_pool_chunks(data: torch.Tensor, chunk_size: int) -> torch.Tensor:
"""Mean-pool complete contextual states, accumulating in FP32."""
chunk_size = min(chunk_size, len(data))
num_full, remainder = divmod(len(data), chunk_size)
full_end = num_full * chunk_size
full_chunks = data[:full_end].reshape(num_full, chunk_size, data.shape[-1])
if not remainder:
return full_chunks.mean(dim=1, dtype=torch.float32)
return torch.cat(
[
full_chunks.mean(dim=1, dtype=torch.float32),
data[full_end:].mean(dim=0, keepdim=True, dtype=torch.float32),
]
)
class TokenPoolerHead(nn.Module, ABC):
@abstractmethod
def get_supported_tasks(self) -> Set[PoolingTask]:
raise NotImplementedError
@abstractmethod
def forward_chunk(
self,
pooled_data: TokenPoolingMethodOutputItem,
pooling_param: PoolingParams,
) -> TokenPoolerHeadOutputItem:
raise NotImplementedError
def forward(
self,
pooled_data: list[TokenPoolingMethodOutputItem],
pooling_metadata: PoolingMetadata,
) -> list[TokenPoolerHeadOutputItem]:
pooling_params = pooling_metadata.pooling_params
if len(pooled_data) != len(pooling_params):
raise ValueError(
f"pooled_data length ({len(pooled_data)}) does not match "
f"pooling_params length ({len(pooling_params)})"
)
return [self.forward_chunk(d, p) for d, p in zip(pooled_data, pooling_params)]
class TokenEmbeddingPoolerHead(TokenPoolerHead):
def __init__(
self,
head_dtype: torch.dtype | str | None = None,
projector: ProjectorFn | None = None,
activation: ActivationFn | None = None,
) -> None:
super().__init__()
self.head_dtype = head_dtype
self.projector = projector
self.activation = activation
def extra_repr(self) -> str:
attrs = []
if self.head_dtype is not None:
attrs.append(f"head_dtype={self.head_dtype}")
if self.projector is not None:
attrs.append("projector=True")
if self.activation is not None:
attrs.append(f"activation={self.activation.__class__.__name__}")
return ", ".join(attrs)
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"token_embed"}
def forward_chunk(
self,
pooled_data: TokenPoolingMethodOutputItem,
pooling_param: PoolingParams,
) -> TokenPoolerHeadOutputItem:
# for unfinished chunked prefill
if pooled_data is None:
return None
if (late_chunking := pooling_param.late_chunking_params) is not None:
means = _mean_pool_chunks(pooled_data, late_chunking.chunk_size)
pooled_data = (
means.to(pooled_data.dtype) if self.head_dtype is None else means
)
if self.head_dtype is not None:
pooled_data = pooled_data.to(self.head_dtype)
# pooled_data shape: [n_tokens, hidden_size]
# Apply ST projector
if self.projector is not None:
embeddings = self.projector(pooled_data)
else:
embeddings = pooled_data
# embeddings shape: [n_tokens, embedding_size]
# for matryoshka representation
if pooling_param.dimensions is not None:
embeddings = embeddings[..., : pooling_param.dimensions]
# for normalize
if self.activation is not None and pooling_param.use_activation:
embeddings = self.activation(embeddings)
# embeddings shape: [n_tokens, embedding_size]
return embeddings
class TokenClassifierPoolerHead(TokenPoolerHead):
def __init__(
self,
classifier: ClassifierFn | None = None,
logit_mean: float | None = None,
logit_sigma: float | None = None,
head_dtype: torch.dtype | str | None = None,
activation: ActivationFn | None = None,
) -> None:
super().__init__()
self.classifier = classifier
self.logit_mean = logit_mean
self.logit_sigma = logit_sigma
self.head_dtype = head_dtype
self.activation = activation
def extra_repr(self) -> str:
attrs = []
if self.head_dtype is not None:
attrs.append(f"head_dtype={self.head_dtype}")
if self.classifier is not None:
attrs.append("classifier=True")
if self.logit_mean is not None:
attrs.append(f"logit_mean={self.logit_mean}")
if self.logit_sigma is not None:
attrs.append(f"logit_sigma={self.logit_sigma}")
if self.activation is not None:
attrs.append(f"activation={self.activation.__class__.__name__}")
return ", ".join(attrs)
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"token_classify"}
def forward_chunk(
self,
pooled_data: TokenPoolingMethodOutputItem,
pooling_param: PoolingParams,
) -> TokenPoolerHeadOutputItem:
# for unfinished chunked prefill
if pooled_data is None:
return None
if self.head_dtype is not None:
pooled_data = pooled_data.to(self.head_dtype)
# hidden_states shape: [n_token, hidden_size]
if self.classifier is not None:
logits = self.classifier(pooled_data)
else:
logits = pooled_data
# logits shape: [n_token, num_labels]
# Affine score calibration: activation((logit - mean) / sigma)
if self.logit_mean is not None:
logits = logits - self.logit_mean
if self.logit_sigma is not None:
logits = logits / self.logit_sigma
if self.activation is not None and pooling_param.use_activation:
logits = self.activation(logits)
# logits shape: [n_token, num_labels]
return logits
@@ -1,149 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Set
from typing import TypeAlias
import torch
import torch.nn as nn
from vllm.config import get_current_vllm_config
from vllm.config.pooler import TokenPoolingType
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.tasks import PoolingTask
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.pool.metadata import PoolingMetadata
TokenPoolingMethodOutputItem: TypeAlias = torch.Tensor | None
class TokenPoolingMethod(nn.Module, ABC):
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"token_embed", "token_classify"}
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return PoolingParamsUpdate()
@abstractmethod
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> list[TokenPoolingMethodOutputItem]:
raise NotImplementedError
class AllPool(TokenPoolingMethod):
def __init__(self):
super().__init__()
vllm_config = get_current_vllm_config()
scheduler_config = vllm_config.scheduler_config
self.enable_chunked_prefill = scheduler_config.enable_chunked_prefill
# With async scheduling, the next step may overwrite the hidden states
# buffer before outputs are copied to CPU, so they must own their storage.
self.clone_finished = bool(scheduler_config.async_scheduling)
def extra_repr(self) -> str:
return (
f"enable_chunked_prefill={self.enable_chunked_prefill}, "
f"clone_finished={self.clone_finished}"
)
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> list[TokenPoolingMethodOutputItem]:
pooling_cursor = pooling_metadata.get_pooling_cursor()
# Use the already-CPU num_scheduled_tokens tensor so `.tolist()`
# doesn't trigger a GPU->CPU sync. torch.split produces the same
# consecutive slices as indexing with first/last per-sequence indices.
hidden_states_lst = list(
torch.split(hidden_states, pooling_cursor.num_scheduled_tokens_cpu.tolist())
)
if not self.enable_chunked_prefill:
if self.clone_finished:
return [hs.clone() for hs in hidden_states_lst]
return hidden_states_lst
pooling_states = pooling_metadata.pooling_states
finished_mask = pooling_cursor.get_finished_mask()
# If chunked_prefill is enabled
# 1. first store the chunked hidden_states in pooling_states.hidden_states_cache
for p, hs_chunk, finished in zip(
pooling_states, hidden_states_lst, finished_mask
):
# Own data retained across steps or returned without concatenation.
needs_owned_storage = not finished or (
self.clone_finished and not p.hidden_states_cache
)
p.hidden_states_cache.append(
hs_chunk.clone() if needs_owned_storage else hs_chunk
)
# 2. Once prefill is finished, send hidden_states_cache to PoolerHead
output_list = list[TokenPoolingMethodOutputItem]()
for p, finished in zip(pooling_states, finished_mask):
if finished:
hidden_states_cache = p.hidden_states_cache
if len(hidden_states_cache) == 1:
output_list.append(hidden_states_cache[0])
else:
output_list.append(torch.concat(hidden_states_cache, dim=0))
p.clean()
else:
output_list.append(None)
return output_list
class StepPool(AllPool):
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return PoolingParamsUpdate(requires_token_ids=True)
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> list[TokenPoolingMethodOutputItem]:
pooled_data_lst = super().forward(hidden_states, pooling_metadata)
# Use the CPU copy of prompt_token_ids so the step_tag_id mask can be
# resolved to indices without a d2h sync from boolean indexing.
prompt_token_ids_cpu = pooling_metadata.get_prompt_token_ids_cpu()
pooling_params = pooling_metadata.pooling_params
pooled_data = list[torch.Tensor | None]()
for data, token_id_cpu, pooling_param in zip(
pooled_data_lst, prompt_token_ids_cpu, pooling_params
):
# for unfinished chunked prefill
if data is None:
pooled_data.append(None)
else:
step_tag_id = pooling_param.step_tag_id
returned_token_ids = pooling_param.returned_token_ids
if returned_token_ids is not None and len(returned_token_ids) > 0:
data = data[:, returned_token_ids]
if step_tag_id is not None:
idx_cpu = (token_id_cpu == step_tag_id).nonzero(as_tuple=True)[0]
idx = async_tensor_h2d(idx_cpu, data.device)
data = data[idx]
pooled_data.append(data)
return pooled_data
def get_tok_pooling_method(pooling_type: TokenPoolingType | str):
if pooling_type == "ALL":
return AllPool()
if pooling_type == "STEP":
return StepPool()
raise NotImplementedError(f"Unknown tokenwise pooling type: {pooling_type!r}")
@@ -1,144 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Set
from typing import TypeAlias
import torch
from vllm.config import PoolerConfig, get_current_vllm_config
from vllm.model_executor.layers.pooler import (
ClassifierFn,
PoolingParamsUpdate,
ProjectorFn,
)
from vllm.model_executor.layers.pooler.abstract import Pooler
from vllm.model_executor.layers.pooler.activations import (
PoolerActivation,
PoolerNormalize,
resolve_classifier_act_fn,
)
from vllm.model_executor.models.adapters import _load_st_projector
from vllm.tasks import POOLING_TASKS, PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata
from .heads import (
TokenClassifierPoolerHead,
TokenEmbeddingPoolerHead,
TokenPoolerHead,
TokenPoolerHeadOutputItem,
)
from .methods import (
TokenPoolingMethod,
TokenPoolingMethodOutputItem,
get_tok_pooling_method,
)
TokenPoolingFn: TypeAlias = Callable[
[torch.Tensor, PoolingMetadata],
list[TokenPoolingMethodOutputItem],
]
TokenPoolingHeadFn: TypeAlias = Callable[
[list[TokenPoolingMethodOutputItem], PoolingMetadata],
list[TokenPoolerHeadOutputItem],
]
TokenPoolerOutput: TypeAlias = list[torch.Tensor | None]
class TokenPooler(Pooler):
"""A layer that pools specific information from hidden states.
This layer does the following:
1. Extracts specific tokens or aggregates data based on pooling method.
2. Postprocesses the output based on pooling head.
3. Returns structured results as `PoolerOutput`.
"""
def __init__(
self,
pooling: TokenPoolingMethod | TokenPoolingFn,
head: TokenPoolerHead | TokenPoolingHeadFn | None = None,
) -> None:
super().__init__()
self.pooling = pooling
self.head = head
def extra_repr(self) -> str:
head_name = self.head.__class__.__name__ if self.head is not None else None
return f"pooling={self.pooling.__class__.__name__}, head={head_name}"
def get_supported_tasks(self) -> Set[PoolingTask]:
tasks = set(POOLING_TASKS)
if isinstance(self.pooling, TokenPoolingMethod):
tasks &= self.pooling.get_supported_tasks()
if isinstance(self.head, TokenPoolerHead):
tasks &= self.head.get_supported_tasks()
return tasks
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
updates = PoolingParamsUpdate()
if isinstance(self.pooling, TokenPoolingMethod):
updates |= self.pooling.get_pooling_updates(task)
return updates
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> TokenPoolerOutput:
pooled_data = self.pooling(hidden_states, pooling_metadata)
if self.head is not None:
pooled_data = self.head(pooled_data, pooling_metadata)
return pooled_data
def pooler_for_token_embed(
pooler_config: PoolerConfig, projector: ProjectorFn | None = None
) -> TokenPooler:
pooling = get_tok_pooling_method(pooler_config.get_tok_pooling_type())
vllm_config = get_current_vllm_config()
model_config = vllm_config.model_config
head = TokenEmbeddingPoolerHead(
head_dtype=model_config.head_dtype,
projector=projector
if projector is not None
else _load_st_projector(model_config),
activation=PoolerNormalize(),
)
return TokenPooler(pooling=pooling, head=head)
def pooler_for_token_classify(
pooler_config: PoolerConfig,
*,
pooling: TokenPoolingMethod | TokenPoolingFn | None = None,
classifier: ClassifierFn | None = None,
act_fn: PoolerActivation | None = None,
) -> TokenPooler:
if pooling is None:
pooling = get_tok_pooling_method(pooler_config.get_tok_pooling_type())
vllm_config = get_current_vllm_config()
model_config = vllm_config.model_config
if model_config.pooler_config is None:
raise ValueError(
"model_config.pooler_config must be set for token classification pooling"
)
head = TokenClassifierPoolerHead(
head_dtype=model_config.head_dtype,
classifier=classifier,
logit_mean=model_config.pooler_config.logit_mean,
logit_sigma=model_config.pooler_config.logit_sigma,
activation=resolve_classifier_act_fn(
model_config, static_num_labels=False, act_fn=act_fn
),
)
return TokenPooler(pooling=pooling, head=head)
@@ -1,294 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from aiter.ops.shuffle import shuffle_weight
from compressed_tensors.quantization import (
QuantizationArgs,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
int4_w4a16_moe_quant_config,
)
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
CompressedTensorsMoEMethod,
)
from vllm.model_executor.utils import set_weight_attrs
logger = init_logger(__name__)
def _pack_shuffled_int8_to_packed_int4_no_perm(x_shuf_i8: torch.Tensor) -> torch.Tensor:
"""Pack a preshuffled int8 tensor (values in [-8, 7]) into packed int4 bytes.
Each contiguous 8-value block [v0..v7] -> 4 bytes:
b0=(v4<<4)|v0, b1=(v5<<4)|v1, b2=(v6<<4)|v2, b3=(v7<<4)|v3.
This matches the 7-op in-kernel unpack sequence and avoids any v_perm.
"""
flat = x_shuf_i8.contiguous().view(-1).to(torch.int16)
assert flat.numel() % 8 == 0
u = (flat & 0xF).to(torch.uint8).view(-1, 8)
out = torch.empty((u.shape[0], 4), device=u.device, dtype=torch.uint8)
out[:, 0] = u[:, 0] | (u[:, 4] << 4)
out[:, 1] = u[:, 1] | (u[:, 5] << 4)
out[:, 2] = u[:, 2] | (u[:, 6] << 4)
out[:, 3] = u[:, 3] | (u[:, 7] << 4)
return out.view(-1).to(torch.int8)
def _unpack_gptq_int32_to_signed_int4(w_int32):
"""Unpack GPTQ int32 [E, K//8, N] to signed int4 values [E, N, K] (as int8).
Shared by both the packed-int4 and bf16-dequant paths.
"""
E = w_int32.shape[0]
# [E, K//8, N] -> transpose -> [E, N, K//8]
w = w_int32.transpose(1, 2).contiguous()
N = w.shape[1]
K_div8 = w.shape[2]
K = K_div8 * 8
# Unpack int32 -> 8 x uint4 values along K
w_expanded = w.unsqueeze(-1).expand(E, N, K_div8, 8) # [E, N, K//8, 8]
shifts = torch.arange(8, device=w.device) * 4 # [0, 4, 8, ..., 28]
nibbles = ((w_expanded >> shifts) & 0xF).to(torch.int8) # [E, N, K//8, 8]
nibbles = nibbles.reshape(E, N, K) # [E, N, K] unsigned int4 as int8
# Convert unsigned [0,15] to signed [-8,7]
signed = nibbles.to(torch.int16) - 8
signed = signed.to(torch.int8) # [E, N, K] signed int4 as int8
return signed
def _gptq_int32_to_flydsl_packed(w_int32):
"""Convert GPTQ int32 [E, K//8, N] to FlyDSL shuffled packed int4 [E, N, K//2].
Steps:
1. Unpack int32 to individual signed int4 values (as int8)
2. Apply FlyDSL preshuffle (on individual int8 values)
3. Pack with FlyDSL's interleaved int4 packing
"""
signed = _unpack_gptq_int32_to_signed_int4(w_int32)
E, N, K = signed.shape
# FlyDSL preshuffle (operates on individual values)
shuffled = shuffle_weight(signed, layout=(16, 16))
# FlyDSL interleaved int4 packing
packed = _pack_shuffled_int8_to_packed_int4_no_perm(shuffled).contiguous()
return packed.view(E, N, K // 2)
class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
def __init__(
self,
weight_quant: QuantizationArgs,
input_quant: QuantizationArgs | None,
moe: FusedMoEConfig,
layer_name: str | None = None,
):
super().__init__(moe)
self.weight_quant = weight_quant
self.input_quant = input_quant
# Extract properties from weight_quant
assert weight_quant.num_bits == 4
self.num_bits = weight_quant.num_bits
self.packed_factor = 32 // weight_quant.num_bits
self.strategy = weight_quant.strategy
# channelwise is not supported by this kernel
assert weight_quant.strategy == "group"
assert weight_quant.group_size == 32
self.group_size = weight_quant.group_size
assert weight_quant.symmetric, (
"Only symmetric quantization is supported for MoE. "
"Try --moe-backend emulation."
)
def create_weights(
self,
layer: torch.nn.Module,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
self.num_experts = num_experts
self.inter_dim = intermediate_size_per_partition
# Will transpose the loaded weight along the
# intermediate and hidden dim sizes. Will
# shard for TP along the transposed dims
extra_weight_attrs.update(
{"is_transposed": True, "quant_method": self.strategy}
)
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size // self.packed_factor,
w13_num_shards * intermediate_size_per_partition,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_packed", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
w2_weight = torch.nn.Parameter(
torch.empty(
num_experts,
intermediate_size_per_partition // self.packed_factor,
hidden_size,
dtype=torch.int32,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
w2_scales_size = intermediate_size_per_partition
if self.strategy == "channel":
num_groups_w2 = num_groups_w13 = 1
self.group_size = -1
else:
num_groups_w2 = w2_scales_size // self.group_size
num_groups_w13 = hidden_size // self.group_size
w13_scale = torch.nn.Parameter(
torch.ones(
num_experts,
num_groups_w13,
w13_num_shards * intermediate_size_per_partition,
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_scale)
set_weight_attrs(w13_scale, extra_weight_attrs)
w2_scale = torch.nn.Parameter(
torch.ones(num_experts, num_groups_w2, hidden_size, dtype=params_dtype),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_scale)
set_weight_attrs(w2_scale, extra_weight_attrs)
w2_weight_shape = torch.nn.Parameter(
torch.empty(num_experts, 2), requires_grad=False
)
layer.register_parameter("w2_weight_shape", w2_weight_shape)
set_weight_attrs(w2_weight_shape, extra_weight_attrs)
w13_weight_shape = torch.nn.Parameter(
torch.empty(num_experts, 2), requires_grad=False
)
layer.register_parameter("w13_weight_shape", w13_weight_shape)
set_weight_attrs(w13_weight_shape, extra_weight_attrs)
layer.a13_scale = None
layer.a2_scale = None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Reconfigure packed weights and scales to match flydsl_w4a16 format
# Convert w13 weights
w13 = layer.w13_weight_packed.data
w13 = _gptq_int32_to_flydsl_packed(w13)
w13 = w13.view(-1).contiguous()
layer.w13_weight_packed = torch.nn.Parameter(w13, requires_grad=False)
# Convert w2 weights
w2 = layer.w2_weight_packed.data
w2 = _gptq_int32_to_flydsl_packed(w2)
w2 = w2.view(-1).contiguous()
layer.w2_weight_packed = torch.nn.Parameter(w2, requires_grad=False)
# Convert scales for FlyDSL:
# per-row: [E, 1, N] -> squeeze -> [E, N]
# groupwise: [E, K//gs, N] -> keep as-is (Opt 0: cache-friendly layout)
w13_scale = layer.w13_weight_scale.data
if self.group_size > 0 and w13_scale.dim() == 3 and w13_scale.shape[1] > 1:
E, G, N = w13_scale.shape
w13_scale = (
w13_scale.view(E, G // 2, 2, N)
.permute(0, 1, 3, 2)
.contiguous()
.view(-1)
.contiguous()
)
elif w13_scale.dim() == 3 and w13_scale.shape[1] == 1:
# Per-row: squeeze [E, 1, N] -> [E, N]
w13_scale = w13_scale.squeeze(1)
layer.w13_weight_scale = torch.nn.Parameter(
w13_scale.contiguous(), requires_grad=False
)
w2_scale = layer.w2_weight_scale.data
if self.group_size > 0 and w2_scale.dim() == 3 and w2_scale.shape[1] > 1:
E, G, N = w2_scale.shape
w2_scale = (
w2_scale.view(E, G // 2, 2, N)
.permute(0, 1, 3, 2)
.contiguous()
.view(-1)
.contiguous()
)
elif w2_scale.dim() == 3 and w2_scale.shape[1] == 1:
# Per-row: squeeze [E, 1, N] -> [E, N]
w2_scale = w2_scale.squeeze(1)
layer.w2_weight_scale = torch.nn.Parameter(
w2_scale.contiguous(), requires_grad=False
)
layer.w13_weight_packed.is_shuffled = True
layer.w2_weight_packed.is_shuffled = True
layer.is_aiter_converted = True
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> FusedMoEQuantConfig | None:
assert self.num_bits == 4
return int4_w4a16_moe_quant_config(
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w1_zp=None,
w2_zp=None,
block_shape=[0, self.group_size],
)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
from vllm.model_executor.layers.fused_moe.fused_flydsl_moe import (
fused_flydsl_moe,
)
assert self.moe_quant_config is not None
return fused_flydsl_moe(
x,
layer.w13_weight_packed,
layer.w2_weight_packed,
self.num_experts,
self.inter_dim,
topk_weights,
topk_ids,
w1_scale=self.moe_quant_config.w1_scale,
w2_scale=self.moe_quant_config.w2_scale,
topk=topk_weights.shape[-1],
group_size=self.group_size,
doweight_stage1=layer.apply_router_weight_on_input,
scale_is_bf16=True,
)
@@ -1,272 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
mxfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsMxfp4,
)
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
MarlinExperts,
)
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsMxFp4,
)
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
is_flashinfer_moe_ep_backend,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
B12X_BACKENDS,
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
Mxfp4MoeBackend,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
make_mxfp4_moe_quant_config,
select_mxfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
CompressedTensorsMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_moe_fp4_layer_for_marlin,
)
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
from vllm.platforms import current_platform
logger = init_logger(__name__)
class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
def __init__(self, moe):
super().__init__(moe)
self.group_size = 32
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
self.experts_cls: type[mk.FusedMoEExperts] | None = None
# Backend selection must match the weight preparation below: CUTLASS
# swizzles scales, b12x and XPU consume checkpoint packing, Marlin
# repacks weights and scales, and the megakernels take the checkpoint
# tensors as they are.
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
if moe.moe_backend == "b12x" or is_flashinfer_moe_ep_backend(moe.moe_backend):
self.mxfp4_backend, experts_cls = select_mxfp4_moe_backend(moe)
assert experts_cls is not None
self.experts_cls = experts_cls
self.use_cutlass_mxfp4 = False
elif self.use_cutlass_mxfp4:
logger.info_once("Using CutlassExpertsMxfp4 for MXFP4 MoE")
self.experts_cls = CutlassExpertsMxfp4
elif current_platform.is_xpu():
self.mxfp4_backend = Mxfp4MoeBackend.XPU
self.experts_cls = XPUExpertsMxFp4
logger.info_once("Using XPUExpertsMxFp4 for MXFP4 MoE on XPU platform")
else:
logger.info_once("Using MarlinExperts for MXFP4 MoE")
self.experts_cls = MarlinExperts
@property
def supports_eplb(self) -> bool:
return self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS
def create_weights(
self,
layer: torch.nn.Module,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
layer.num_experts = num_experts
layer.params_dtype = params_dtype
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
self.moe.w13_num_shards * intermediate_size_per_partition,
# 2 fp4 items are packed in the input dimension
hidden_size // 2,
requires_grad=False,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_packed", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
w2_weight = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
# 2 fp4 items are packed in the input dimension
intermediate_size_per_partition // 2,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
w13_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
self.moe.w13_num_shards * intermediate_size_per_partition,
# 2 fp4 items are packed in the input dimension
hidden_size // self.group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
extra_weight_attrs.update(
{"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
)
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
w2_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
# 2 fp4 items are packed in the input dimension
intermediate_size_per_partition // self.group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> FusedMoEQuantConfig | None:
if self.use_cutlass_mxfp4:
# W4A4: both weights and activations quantized to MXFP4
return mxfp4_moe_quant_config(
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
)
else:
# b12x selects W4A8 or W4A16; XPU uses W4A4; Marlin uses W4A16.
return make_mxfp4_moe_quant_config(
mxfp4_backend=self.mxfp4_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
layer=layer,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
layer.w13_weight = torch.nn.Parameter(
layer.w13_weight_packed.data, requires_grad=False
)
delattr(layer, "w13_weight_packed")
layer.w2_weight = torch.nn.Parameter(
layer.w2_weight_packed.data, requires_grad=False
)
delattr(layer, "w2_weight_packed")
if self.use_cutlass_mxfp4:
# Swizzle weight scales from flat checkpoint layout [E, N, K//32]
# to CUTLASS tiled layout [E, numMTiles*numKTiles*512].
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
swizzle_mxfp4_scales,
)
E = layer.w13_weight_scale.shape[0]
w13_N = layer.w13_weight_scale.shape[1]
w13_scale_K = layer.w13_weight_scale.shape[2]
w13_K = w13_scale_K * 32
w2_M = layer.w2_weight_scale.shape[1]
w2_scale_N = layer.w2_weight_scale.shape[2]
w2_N = w2_scale_N * 32
swizzled_w13 = []
swizzled_w2 = []
for e_idx in range(E):
s13 = layer.w13_weight_scale[e_idx]
sw13 = swizzle_mxfp4_scales(s13, w13_N, w13_K)
swizzled_w13.append(sw13.reshape(w13_N, w13_scale_K))
s2 = layer.w2_weight_scale[e_idx]
sw2 = swizzle_mxfp4_scales(s2, w2_M, w2_N)
swizzled_w2.append(sw2.reshape(w2_M, w2_scale_N))
layer.w13_weight_scale = torch.nn.Parameter(
torch.stack(swizzled_w13), requires_grad=False
)
layer.w2_weight_scale = torch.nn.Parameter(
torch.stack(swizzled_w2), requires_grad=False
)
elif self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
w13, w2, w13_scale, w2_scale, _, _ = (
convert_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend=self.mxfp4_backend,
layer=layer,
w13_weight=layer.w13_weight,
w2_weight=layer.w2_weight,
w13_weight_scale=layer.w13_weight_scale,
w2_weight_scale=layer.w2_weight_scale,
)
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
pass
else:
logger.warning_once(
"Your GPU does not have native support for FP4 computation "
"but FP4 quantization is being used. Weight-only FP4 "
"compression will be used leveraging the Marlin kernel. "
"This may degrade performance for compute-heavy workloads."
)
prepare_moe_fp4_layer_for_marlin(layer)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None:
assert self.experts_cls is not None
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
mxfp4_backend=self.mxfp4_backend,
routing_tables=layer._expert_routing_tables(),
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_weight,
layer.w2_weight,
topk_weights,
topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -1,34 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
_PACKED_WEIGHTS = ("w13_weight", "w2_weight")
class CompressedTensorsW4A8Mxfp4MoEMethod(Mxfp4MoEMethod):
"""MXFP4 weights with dynamic FP8 activations in compressed-tensors format.
Only the checkpoint naming and the declared activation scheme differ from
the native MXFP4 path, so backend selection, weight conversion and
execution are inherited.
"""
# The packed -> unpacked rename means a processed state dict no longer
# matches the parameters registered by create_weights.
supports_pre_processed_weights = False
def create_weights(self, layer: RoutedExperts, *args, **kwargs):
super().create_weights(layer, *args, **kwargs)
for name in _PACKED_WEIGHTS:
param = getattr(layer, name)
delattr(layer, name)
layer.register_parameter(f"{name}_packed", param)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
for name in _PACKED_WEIGHTS:
param = getattr(layer, f"{name}_packed")
delattr(layer, f"{name}_packed")
layer.register_parameter(name, param)
super().process_weights_after_loading(layer)
@@ -1,208 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
select_mxfp8_moe_backend,
)
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe import ( # noqa: E501
CompressedTensorsMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
)
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
"""Compressed-tensors MoE method for pre-quantized MXFP8 (W8A8) checkpoints.
Loads FP8 (E4M3) weights with E8M0 uint8 per-group scales (group_size=32)
from checkpoint. Activations are dynamically quantized to MXFP8 at runtime.
Supports FlashInfer TRT-LLM and Marlin backends (auto-selected).
"""
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_block_size = [1, MXFP8_BLOCK_SIZE]
self.fp8_backend, self.experts_cls = select_mxfp8_moe_backend(config=self.moe)
def create_weights(
self,
layer: torch.nn.Module,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
layer.num_experts = num_experts
layer.params_dtype = params_dtype
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
hidden_size,
dtype=MXFP8_VALUE_DTYPE,
),
requires_grad=False,
)
layer.register_parameter("w13_weight", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
w2_weight = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition,
dtype=MXFP8_VALUE_DTYPE,
),
requires_grad=False,
)
layer.register_parameter("w2_weight", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
w13_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
hidden_size // MXFP8_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
extra_weight_attrs.update(
{"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
)
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
w2_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition // MXFP8_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
layer.w13_input_scale = None
layer.w2_input_scale = None
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
layer.weight_block_size = self.weight_block_size
w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format(
fp8_backend=self.fp8_backend,
layer=layer,
w13=layer.w13_weight,
w2=layer.w2_weight,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w13_input_scale=layer.w13_input_scale,
w2_input_scale=layer.w2_input_scale,
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None:
assert self.experts_cls is not None
self.moe_kernel = make_fp8_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> FusedMoEQuantConfig | None:
return make_fp8_moe_quant_config(
fp8_backend=self.fp8_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
a1_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
block_shape=self.weight_block_size,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
layer=layer,
)
def apply_monolithic(
self,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
x,
layer.w13_weight,
layer.w2_weight,
router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_weight,
layer.w2_weight,
topk_weights,
topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -5,8 +5,8 @@ from .compressed_tensors_scheme import CompressedTensorsScheme
from .compressed_tensors_w4a4_mxfp4 import CompressedTensorsW4A4Mxfp4
from .compressed_tensors_w4a4_nvfp4 import CompressedTensorsW4A4Fp4
from .compressed_tensors_w4a8_fp8 import CompressedTensorsW4A8Fp8
from .compressed_tensors_w4a8_int import CompressedTensorsW4A8Int
from .compressed_tensors_w4a8_mxfp4 import CompressedTensorsW4A8Mxfp4
from .compressed_tensors_w4a8_int import CompressedTensorsW4A8Int
from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
from .compressed_tensors_w8a8_mxfp8 import CompressedTensorsW8A8Mxfp8
@@ -23,12 +23,12 @@ __all__ = [
"CompressedTensorsW8A16Fp8",
"CompressedTensorsW8A8Int8",
"CompressedTensorsW8A8Fp8",
"CompressedTensorsW4A4Fp4",
"CompressedTensorsW4A4Mxfp4",
"CompressedTensorsW4A8Mxfp4",
"CompressedTensorsW4A4Fp4",
"CompressedTensorsW8A8Mxfp8",
"CompressedTensorsW4A8Int",
"CompressedTensorsW4A8Fp8",
"CompressedTensorsW8A8Mxfp8",
"CompressedTensorsWNA4Int",
"CompressedTensorsWNA8Int",
]
@@ -1,26 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TurboQuant: KV-cache quantization for vLLM.
Hadamard rotation + per-coordinate Lloyd-Max scalar quantization for
keys, uniform quantization for values.
The core algorithmic pattern implemented for key quantization (Hadamard
rotation followed by deterministic scalar quantization and
re-normalization) was originally established in DRIVE (Vargaftik et al.,
NeurIPS 2021) and EDEN (Vargaftik et al., ICML 2022). This formulation is
also mathematically equivalent to the scalar case of the HIGGS
quantization method (Malinovskii et al., "Pushing the Limits of Large
Language Model Quantization via the Linearity Theorem", NAACL 2025;
preprint arXiv:2411.17525), which subsequently generalized these concepts.
A first application of this approach to KV-cache compression is in "Cache
Me If You Must: Adaptive Key-Value Quantization for Large Language Models"
(Shutova et al., ICML 2025; preprint arXiv:2501.19392). All of these
foundational and application references pre-date the TurboQuant paper
(Zandieh et al., ICLR 2026).
"""
from vllm.model_executor.layers.quantization.turboquant.config import TurboQuantConfig
__all__ = ["TurboQuantConfig"]
@@ -1,87 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Lloyd-Max optimal scalar quantizer for TurboQuant.
After rotating a d-dimensional unit vector by a random orthogonal matrix,
each coordinate approximately follows N(0, 1/d) for d >= 64.
We solve the Lloyd-Max conditions to find optimal centroids.
Based on: turboquant-pytorch/lloyd_max.py (Zandieh et al.)
"""
import math
from functools import lru_cache
import torch
def _gaussian_pdf(x: float, sigma2: float) -> float:
return (1.0 / math.sqrt(2 * math.pi * sigma2)) * math.exp(-x * x / (2 * sigma2))
def _trapz(f, a: float, b: float, n: int = 200) -> float:
"""Trapezoidal numerical integration (replaces scipy.integrate.quad)."""
h = (b - a) / n
result = 0.5 * (f(a) + f(b))
for i in range(1, n):
result += f(a + i * h)
return result * h
def solve_lloyd_max(
d: int,
bits: int,
max_iter: int = 200,
tol: float = 1e-10,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Solve Lloyd-Max optimal quantizer for N(0, 1/d) distribution.
Args:
d: Vector dimension (determines variance = 1/d).
bits: Number of quantization bits.
max_iter: Maximum Lloyd-Max iterations.
tol: Convergence tolerance.
Returns:
centroids: Sorted tensor of 2^bits optimal centroids.
boundaries: Sorted tensor of 2^bits - 1 decision boundaries.
"""
n_levels = 2**bits
sigma2 = 1.0 / d
sigma = math.sqrt(sigma2)
def pdf(x):
return _gaussian_pdf(x, sigma2)
lo, hi = -3.5 * sigma, 3.5 * sigma
centroids = [lo + (hi - lo) * (i + 0.5) / n_levels for i in range(n_levels)]
for _ in range(max_iter):
boundaries = [
(centroids[i] + centroids[i + 1]) / 2.0 for i in range(n_levels - 1)
]
edges = [lo * 3] + boundaries + [hi * 3]
new_centroids = []
for i in range(n_levels):
a, b = edges[i], edges[i + 1]
num = _trapz(lambda x: x * pdf(x), a, b)
den = _trapz(pdf, a, b)
new_centroids.append(num / den if den > 1e-15 else centroids[i])
if max(abs(new_centroids[i] - centroids[i]) for i in range(n_levels)) < tol:
break
centroids = new_centroids
boundaries = [(centroids[i] + centroids[i + 1]) / 2.0 for i in range(n_levels - 1)]
return (
torch.tensor(centroids, dtype=torch.float32),
torch.tensor(boundaries, dtype=torch.float32),
)
@lru_cache(maxsize=32)
def get_centroids(d: int, bits: int) -> torch.Tensor:
"""Get precomputed Lloyd-Max centroids (cached)."""
centroids, _ = solve_lloyd_max(d, bits)
return centroids
@@ -1,261 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TurboQuant configuration."""
from __future__ import annotations
import logging
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from vllm.config import ModelConfig
logger = logging.getLogger(__name__)
# Named TQ presets: each maps to frozen config parameters.
# key_quant_bits: 8 = FP8 keys, 3-4 = MSE (Lloyd-Max) quantized keys.
# value_quant_bits: 3-4 = uniform quantized values.
TQ_PRESETS: dict[str, dict] = {
"turboquant_k8v4": {
"key_quant_bits": 8,
"value_quant_bits": 4,
"norm_correction": False,
},
"turboquant_4bit_nc": {
"key_quant_bits": 4,
"value_quant_bits": 4,
"norm_correction": True,
},
"turboquant_k3v4_nc": {
"key_quant_bits": 3,
"value_quant_bits": 4,
"norm_correction": True,
},
"turboquant_3bit_nc": {
"key_quant_bits": 3,
"value_quant_bits": 3,
"norm_correction": True,
},
}
@dataclass
class TurboQuantConfig:
"""Configuration for TurboQuant KV-cache quantization.
Applies Hadamard rotation followed by per-coordinate Lloyd-Max scalar
quantization for keys, and uniform quantization for values.
Historical note: the core algorithmic pattern implemented for key
quantization (Hadamard rotation followed by deterministic scalar
quantization and re-normalization) was originally established in DRIVE
(Vargaftik et al., NeurIPS 2021) and EDEN (Vargaftik et al., ICML
2022). This formulation is also mathematically equivalent to the
scalar case of the HIGGS quantization method (Malinovskii et al.,
"Pushing the Limits of Large Language Model Quantization via the
Linearity Theorem", NAACL 2025; preprint arXiv:2411.17525), which
subsequently generalized these concepts.
A first application of this approach to KV-cache compression is in
"Cache Me If You Must: Adaptive Key-Value Quantization for Large
Language Models" (Shutova et al., ICML 2025; preprint
arXiv:2501.19392). All of these foundational and application
references pre-date the TurboQuant paper (Zandieh et al., ICLR 2026).
QJL is intentionally omitted: community consensus (5+ independent
groups) found it hurts attention quality by amplifying variance
through softmax.
Named presets (use via --kv-cache-dtype):
turboquant_k8v4: FP8 keys + 4-bit values, 2.6x, +1.17% PPL
turboquant_4bit_nc: 4-bit MSE keys + 4-bit values + NC, 3.8x, +2.71%
turboquant_k3v4_nc: 3-bit MSE keys + 4-bit values + NC, ~3.5x, +10.63%
turboquant_3bit_nc: 3-bit MSE keys + 3-bit values + NC, 4.9x, +20.59%
Args:
head_dim: Attention head dimension (e.g. 64, 96, 128).
key_quant_bits: Bits for key quantization. 8 = FP8 keys (no
rotation/MSE). 3-4 = Lloyd-Max MSE quantized keys.
value_quant_bits: Bits per value dimension for uniform quantization.
3 = 8 levels, 4 = 16 levels (default).
norm_correction: Re-normalize centroid vectors to unit norm before
inverse rotation during dequant. Fixes quantization-induced norm
distortion, improving PPL by ~0.8% at 4-bit.
"""
head_dim: int = 128
key_quant_bits: int = 3 # 3-4 = MSE keys, 8 = FP8 keys
value_quant_bits: int = 4 # 3-4 = uniform quantized values
seed: int = 42 # kept for backward compatibility; no longer used internally
norm_correction: bool = False
@property
def key_fp8(self) -> bool:
"""Whether keys are stored as FP8 — no rotation/quantization needed."""
return self.key_quant_bits == 8
@property
def mse_bits(self) -> int:
"""MSE quantizer bit-width (determines centroid count: 2^mse_bits).
For MSE key modes, equals key_quant_bits.
For FP8 key mode, falls back to value_quant_bits (centroids are still
needed for continuation-prefill dequant and decode kernel params).
"""
if self.key_fp8:
return self.value_quant_bits
return self.key_quant_bits
@property
def key_mse_bits(self) -> int:
"""MSE bits actually used for key quantization (0 if FP8 keys)."""
if self.key_fp8:
return 0
return self.key_quant_bits
@property
def centroid_bits(self) -> int:
"""Bits for centroid generation — always non-zero."""
return self.mse_bits
@property
def n_centroids(self) -> int:
return 2**self.mse_bits
@property
def key_packed_size(self) -> int:
"""Packed bytes for a single KEY vector.
FP8 mode (key_quant_bits=8):
head_dim bytes (1 byte per element, no overhead).
TQ mode:
- MSE indices: ceil(head_dim * key_mse_bits / 8) bytes
- vec_norm: 2 bytes (float16)
"""
if self.key_fp8:
return self.head_dim # 1 byte per element
mse_bytes = math.ceil(self.head_dim * self.key_mse_bits / 8)
norm_bytes = 2 # vec_norm fp16
return mse_bytes + norm_bytes
@property
def effective_value_quant_bits(self) -> int:
"""Actual bits used for value storage."""
return self.value_quant_bits
@property
def value_packed_size(self) -> int:
"""Packed bytes for a single VALUE vector.
Uniform quantization: ceil(head_dim * bits / 8) + 4 bytes (scale + zero fp16).
"""
data_bytes = math.ceil(self.head_dim * self.value_quant_bits / 8)
return data_bytes + 4 # +2 scale(fp16) +2 zero(fp16)
@property
def slot_size(self) -> int:
"""Total packed bytes per head per position (key + value combined).
Layout: [key_packed | value_packed]
"""
return self.key_packed_size + self.value_packed_size
@property
def slot_size_aligned(self) -> int:
"""Slot size rounded up to next even number.
Even-number is required so effective_head_size = slot_size_aligned // 2
is integral.
"""
s = self.slot_size
return s + (s % 2) # round up to even
@staticmethod
def get_boundary_skip_layers(
model_config: ModelConfig,
n: int = 2,
) -> list[str]:
"""Layer indices to skip TQ compression (boundary protection).
For hybrid models (attention + Mamba/linear-attention), boundary
protection is disabled — hybrids typically have only 8-12
full-attention layers and a hard n=2 on each side would cover
~40 % of them. The dense GSM8K baselines that motivate n=2
don't apply to hybrids.
For dense models, skips first N and last N attention layers.
Empirically required for aggressive presets (k3v4_nc, 3bit_nc)
— without it GSM8K drops ~30 points on Qwen3-4B.
"""
if model_config.is_hybrid:
attn_indices = _get_full_attention_layer_indices(model_config)
if not attn_indices:
raise NotImplementedError(
"TurboQuant KV cache requires identifiable "
"full-attention layers, but none were found in "
"the hybrid model config."
)
logger.info("TQ hybrid: full-attention layers %s", attn_indices)
return []
num_layers = model_config.hf_text_config.num_hidden_layers
if n <= 0 or num_layers <= 0:
return []
n = min(n, num_layers // 2) # don't skip more than half
first = list(range(n))
last = list(range(num_layers - n, num_layers))
# Deduplicate (if num_layers <= 2*n)
indices = sorted(set(first + last))
return [str(i) for i in indices]
@staticmethod
def from_cache_dtype(cache_dtype: str, head_dim: int) -> TurboQuantConfig:
"""Create config from a named preset.
Valid presets: turboquant_k8v4, turboquant_4bit_nc, etc.
"""
if cache_dtype not in TQ_PRESETS:
valid = ", ".join(TQ_PRESETS.keys())
raise ValueError(
f"Unknown TurboQuant cache dtype: {cache_dtype!r}. "
f"Valid presets: {valid}"
)
preset = TQ_PRESETS[cache_dtype]
return TurboQuantConfig(
head_dim=head_dim,
key_quant_bits=preset["key_quant_bits"],
value_quant_bits=preset["value_quant_bits"],
norm_correction=preset["norm_correction"],
)
def _get_full_attention_layer_indices(model_config: ModelConfig) -> list[int]:
"""Global indices of full-attention layers in a hybrid model.
Covers the conventions used across vLLM: ``layer_types`` (Qwen3.5/Next),
``layers_block_type`` (Jamba/Zamba2), ``attn_type_list`` (Minimax).
"""
text_cfg = model_config.hf_text_config
hf_cfg = model_config.hf_config
layer_types = getattr(text_cfg, "layer_types", None)
if layer_types is not None:
return [
i for i, t in enumerate(layer_types) if t in ("full_attention", "attention")
]
layers_block_type = getattr(text_cfg, "layers_block_type", None)
if layers_block_type is not None:
return [
i for i, t in enumerate(layers_block_type) if t in ("attention", "hybrid")
]
attn_type_list = getattr(hf_cfg, "attn_type_list", None)
if attn_type_list is not None:
return [i for i, t in enumerate(attn_type_list) if t == 1]
return []
@@ -1,207 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility functions used for tests and benchmarks."""
import numpy as np
import torch
from vllm import _custom_ops as ops
from vllm.scalar_type import ScalarType, scalar_types
from .marlin_utils import GPTQ_MARLIN_TILE, marlin_permute_scales, marlin_zero_points
from .quant_utils import (
get_pack_factor,
gptq_quantize_weights,
quantize_weights,
)
class MarlinWorkspace:
def __init__(self, out_features, min_thread_n, max_parallel):
assert out_features % min_thread_n == 0, (
"out_features = {} is indivisible by min_thread_n = {}".format(
out_features, min_thread_n
)
)
max_workspace_size = (out_features // min_thread_n) * max_parallel
self.scratch = torch.zeros(max_workspace_size, dtype=torch.int, device="cuda")
def marlin_permute_weights(
q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE, is_a_8bit=False
):
assert q_w.shape == (size_k, size_n)
assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
if is_a_8bit:
# Permute weights to 32x32 marlin tiles
q_w = q_w.reshape((size_k // (tile * 2), tile * 2, size_n // tile, tile))
else:
# Permute weights to 16x64 marlin tiles
q_w = q_w.reshape((size_k // tile, tile, size_n // tile, tile))
q_w = q_w.permute((0, 2, 1, 3))
q_w = q_w.reshape((size_k // tile, size_n * tile))
q_w = q_w.reshape((-1, perm.numel()))[:, perm].reshape(q_w.shape)
return q_w
def marlin_weights(q_w, size_k, size_n, num_bits, perm, is_a_8bit=False):
# Permute
q_w = marlin_permute_weights(q_w, size_k, size_n, perm, is_a_8bit=is_a_8bit)
# Pack
pack_factor = get_pack_factor(num_bits)
orig_device = q_w.device
q_w = q_w.cpu().numpy().astype(np.uint32)
q_packed = np.zeros((q_w.shape[0], q_w.shape[1] // pack_factor), dtype=np.uint32)
for i in range(pack_factor):
q_packed |= q_w[:, i::pack_factor] << num_bits * i
q_packed = torch.from_numpy(q_packed.astype(np.int32)).to(orig_device)
return q_packed
def get_weight_perm(num_bits: int, is_a_8bit: bool = False):
perm_list: list[int] = []
if is_a_8bit:
for i in range(32):
perm1 = []
col = i // 4
for block in [0, 1]:
for row in [
4 * (i % 4),
4 * (i % 4) + 1,
4 * (i % 4) + 2,
4 * (i % 4) + 3,
4 * (i % 4 + 4),
4 * (i % 4 + 4) + 1,
4 * (i % 4 + 4) + 2,
4 * (i % 4 + 4) + 3,
]:
perm1.append(16 * row + col + 8 * block)
for j in range(2):
perm_list.extend([p + 512 * j for p in perm1])
else:
for i in range(32):
perm1 = []
col = i // 4
for block in [0, 1]:
for row in [
2 * (i % 4),
2 * (i % 4) + 1,
2 * (i % 4 + 4),
2 * (i % 4 + 4) + 1,
]:
perm1.append(16 * row + col + 8 * block)
for j in range(4):
perm_list.extend([p + 256 * j for p in perm1])
perm = np.array(perm_list)
if num_bits == 4:
if is_a_8bit: # noqa: SIM108
interleave = np.array([0, 4, 1, 5, 2, 6, 3, 7])
else:
interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7])
elif num_bits == 8:
if is_a_8bit: # noqa: SIM108
interleave = np.array([0, 1, 2, 3])
else:
interleave = np.array([0, 2, 1, 3])
else:
raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
perm = torch.from_numpy(perm)
return perm
def marlin_quantize(
w: torch.Tensor,
quant_type: ScalarType,
group_size: int,
input_dtype: torch.dtype | None = None,
):
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
size_k, size_n = w.shape
num_bits = quant_type.size_bits
# Normalize group_size
if group_size == -1:
group_size = size_k
assert group_size <= size_k
w_ref, q_w, s = gptq_quantize_weights(w, quant_type, group_size)
# Reformat to marlin
weight_perm = get_weight_perm(num_bits, is_a_8bit)
marlin_q_w = marlin_weights(
q_w, size_k, size_n, num_bits, weight_perm, is_a_8bit=is_a_8bit
)
marlin_s = marlin_permute_scales(s, size_k, size_n, group_size, is_a_8bit=is_a_8bit)
if input_dtype == torch.float8_e4m3fn and quant_type == scalar_types.uint4b8:
ops.marlin_int4_fp8_preprocess(marlin_q_w, inplace=True)
marlin_s = marlin_s * 512
# Create result
res_list = [w_ref, marlin_q_w, marlin_s]
for i in range(len(res_list)):
res_list[i] = res_list[i].to(w.device)
return res_list
def awq_marlin_quantize(
w: torch.Tensor,
quant_type: ScalarType,
group_size: int,
input_dtype: torch.dtype | None = None,
):
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
size_k, size_n = w.shape
# Normalize group_size
if group_size == -1:
group_size = size_k
assert group_size <= size_k
# Detect num groups
assert size_k % group_size == 0
num_groups = size_k // group_size
# Quantize with zp
w_ref, q_w, s, zp = quantize_weights(w, quant_type, group_size, zero_points=True)
if input_dtype == torch.float8_e4m3fn and quant_type == scalar_types.uint4:
repeated_zp = zp.repeat_interleave(group_size, 0)
q_w_old = q_w
q_w = q_w_old - repeated_zp
q_w[q_w < 0] = 15 - q_w_old[q_w < 0]
s = s * 512
# Reformat to marlin
weight_perm = get_weight_perm(quant_type.size_bits, is_a_8bit)
marlin_q_w = marlin_weights(
q_w, size_k, size_n, quant_type.size_bits, weight_perm, is_a_8bit=is_a_8bit
)
marlin_s = marlin_permute_scales(s, size_k, size_n, group_size, is_a_8bit=is_a_8bit)
marlin_zp = marlin_zero_points(
zp, num_groups, size_n, quant_type.size_bits, is_a_8bit=is_a_8bit
)
# Create result
res_list = [w_ref, marlin_q_w, marlin_s, marlin_zp]
for i in range(len(res_list)):
res_list[i] = res_list[i].to(w.device)
return res_list
-279
View File
@@ -1,279 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py
# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
#
# Copyright 2023 The Qwen team.
# Copyright 2023 The vLLM team.
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Shared resampler perceiver network used in multimodal models and
related helpers for sincos positional embeddings.
Example models: Qwen (Qwen-VL), MiniCPM-V 2.0
"""
import math
from collections.abc import Callable
from functools import partial
import numpy as np
import torch
import torch.nn.functional as F
from torch import nn
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
def get_abs_pos(abs_pos: torch.Tensor, tgt_size: torch.Tensor | int) -> torch.Tensor:
# abs_pos: L, C
# tgt_size: (H, W)
# return: M, C
src_size = int(math.sqrt(abs_pos.size(0)))
dtype = abs_pos.dtype
if isinstance(tgt_size, int):
tgt_size = (tgt_size, tgt_size)
if src_size == tgt_size[0] and src_size == tgt_size[1]:
return abs_pos
return (
F.interpolate(
abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
size=(tgt_size[0], tgt_size[1]),
mode="bicubic",
align_corners=False,
)
.permute(0, 2, 3, 1)
.flatten(0, 2)
.to(dtype=dtype)
)
# sin/cos positional embedding helpers are adapted from:
# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
def get_1d_sincos_pos_embed_from_grid(
embed_dim: int, pos: np.ndarray, version: tuple[int, int] = (2, 0)
) -> torch.Tensor:
"""embed_dim: output dimension for each position
pos: a list of positions to be encoded: size (M,) / (H, W)
out: (M, D) / (H, W, D)
"""
assert embed_dim % 2 == 0
omega = np.arange(embed_dim // 2, dtype=np.float32)
omega /= embed_dim / 2.0
omega = 1.0 / 10000**omega # (D/2,)
if version == (2, 0):
pos = pos.reshape(-1) # (M,)
out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product
emb_sin = np.sin(out) # (M, D/2)
emb_cos = np.cos(out) # (M, D/2)
emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
else:
out = np.einsum("hw,d->hwd", pos, omega) # (H, W, D/2), outer product
emb_sin = np.sin(out) # (H, W, D/2)
emb_cos = np.cos(out) # (H, W, D/2)
emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
return emb
def get_2d_sincos_pos_embed_from_grid(
embed_dim: int, grid: np.ndarray, version: tuple[int, int] = (2, 0)
) -> torch.Tensor:
assert embed_dim % 2 == 0
# use half of dimensions to encode grid_h
emb_h = get_1d_sincos_pos_embed_from_grid(
embed_dim // 2, grid[0], version
) # (H*W, D/2) or (H, W, D/2)
emb_w = get_1d_sincos_pos_embed_from_grid(
embed_dim // 2, grid[1], version
) # (H*W, D/2) or (H, W, D/2)
if version == (2, 0):
emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
else:
emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D)
return emb
def get_2d_sincos_pos_embed(
embed_dim: int,
grid_size: int | tuple[int, int],
cls_token: bool = False,
version: tuple[int, int] = (2, 0),
) -> torch.Tensor:
"""grid_size: int of the grid height and width
return:
pos_embed: [grid_size*grid_size, embed_dim] or
[1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
"""
if isinstance(grid_size, int):
grid_h_size, grid_w_size = grid_size, grid_size
else:
grid_h_size, grid_w_size = grid_size[0], grid_size[1]
grid_h = np.arange(grid_h_size, dtype=np.float32)
grid_w = np.arange(grid_w_size, dtype=np.float32)
grid = np.meshgrid(grid_w, grid_h) # here w goes first
grid = np.stack(grid, axis=0)
assert isinstance(grid, np.ndarray) and grid.shape == (2, grid_h_size, grid_w_size)
if version == (2, 0):
grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
if cls_token:
pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
else:
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
return pos_embed
class BaseResampler(nn.Module):
"""A 2D perceiver-resampler network with one cross attention layers by
(grid_size**2) learnable queries and 2d sincos pos_emb.
Outputs:
A tensor with the shape of (grid_size**2, embed_dim)
"""
def __init__(
self,
num_queries: int,
embed_dim: int,
num_heads: int,
kv_dim: int | None = None,
norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
do_post_projection: bool = True,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.num_queries = num_queries
self.embed_dim = embed_dim
self.num_heads = num_heads
self.query = nn.Parameter(torch.empty(self.num_queries, embed_dim))
if kv_dim is not None and kv_dim != embed_dim:
self.kv_proj = ReplicatedLinear(
kv_dim,
embed_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.kv_proj",
)
else:
# Maintain the same return value with ReplicatedLinear.forward
self.kv_proj = lambda *args, **kwargs: ( # type: ignore # noqa
nn.Identity()(*args, **kwargs),
None,
)
self.attn = nn.MultiheadAttention(embed_dim, num_heads)
self.ln_q = norm_layer(embed_dim)
self.ln_kv = norm_layer(embed_dim)
self.do_post_projection = do_post_projection
if self.do_post_projection:
self.ln_post = norm_layer(embed_dim)
data = (embed_dim**-0.5) * torch.empty(embed_dim, embed_dim)
self.proj = nn.Parameter(data=data)
def _repeat(self, query, N: int):
return query.unsqueeze(1).repeat(1, N, 1)
class Resampler2(BaseResampler):
"""Resampler-perceiver network to be used for a variety of model types,
e.g., Qwen-vl / Minicpmv 2.0. The main difference is the addition of the
do_post_projection arg, which indicates whether or not there should be
a post layer normalization and projector after the attention. This is
present in minicpmv2.0, but not qwen-vl.
"""
def __init__(
self,
grid_size: int,
embed_dim: int,
num_heads: int,
kv_dim: int | None = None,
norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
adaptive: bool = False,
do_post_projection: bool = True,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__(
grid_size**2,
embed_dim,
num_heads,
kv_dim,
norm_layer,
do_post_projection=do_post_projection,
quant_config=quant_config,
prefix=prefix,
)
self.adaptive = adaptive
pos_embed_arr = get_2d_sincos_pos_embed(embed_dim, grid_size, version=(2, 0))
self.pos_embed = nn.Parameter(
torch.from_numpy(pos_embed_arr).requires_grad_(False)
)
def forward(
self,
x: torch.Tensor,
tgt_sizes: torch.Tensor | None = None,
attn_mask: torch.Tensor | None = None,
) -> torch.Tensor:
if tgt_sizes is None:
tgt_sizes = int(math.sqrt(x.size(1)))
if self.adaptive:
pos_embed_arr = get_2d_sincos_pos_embed(
self.embed_dim, tgt_sizes, version=(2, 0)
)
pos_embed = torch.from_numpy(pos_embed_arr).to(
device=x.device, dtype=x.dtype
)
else:
pos_embed = get_abs_pos(self.pos_embed, tgt_sizes).to(
device=x.device, dtype=x.dtype
)
x, _ = self.kv_proj(x)
x = self.ln_kv(x).permute(1, 0, 2)
N = x.shape[1]
q = self.ln_q(self.query)
out = self.attn(
self._repeat(q, N) + self.pos_embed.unsqueeze(1),
x + pos_embed.unsqueeze(1),
x,
attn_mask=attn_mask,
)[0]
x = out.permute(1, 0, 2)
if self.do_post_projection:
x = self.ln_post(x)
x = x @ self.proj
return x
@@ -1,93 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from .mrope import MRotaryEmbedding
class Ernie4_5_VLRotaryEmbedding(MRotaryEmbedding):
"""3D rotary positional embedding. 3D is t:time h:height w:width."""
def forward_native( # type: ignore[override]
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
assert positions.ndim == 1 or positions.ndim == 2
assert key is not None
num_tokens = positions.shape[-1]
cos_sin = self.cos_sin_cache[positions]
cos, sin = cos_sin.chunk(2, dim=-1)
if positions.ndim == 2:
assert self.mrope_section
section_h = self.mrope_section[0] # 22
section_w = self.mrope_section[1] # 22
section_t = self.mrope_section[2] # 20
assert section_h == section_w
# Split according to [h w h w h w h w... t t t...]
section_cos_t = cos[..., -section_t:]
section_cos_h = cos[..., : section_h + section_w : 2]
section_cos_w = cos[..., 1 : section_h + section_w : 2]
cos_t, cos_h, cos_w = section_cos_t[0], section_cos_h[1], section_cos_w[2]
cos_hw = torch.stack([cos_h, cos_w], dim=-1).reshape(
cos_h.shape[:-1] + (cos_h.shape[-1] * 2,)
)
cos = torch.cat([cos_hw, cos_t], dim=-1)
section_sin_t = sin[..., -section_t:]
section_sin_h = sin[..., : section_h + section_w : 2]
section_sin_w = sin[..., 1 : section_h + section_w : 2]
sin_t, sin_h, sin_w = section_sin_t[0], section_sin_h[1], section_sin_w[2]
sin_hw = torch.stack([sin_h, sin_w], dim=-1).reshape(
sin_h.shape[:-1] + (sin_h.shape[-1] * 2,)
)
sin = torch.cat([sin_hw, sin_t], dim=-1)
query_shape = query.shape
query = query.view(num_tokens, -1, self.head_size)
query_rot = query[..., : self.rotary_dim]
query_pass = query[..., self.rotary_dim :]
query_rot = self.apply_rotary_emb.forward_native(
query_rot,
cos,
sin,
)
query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
key_shape = key.shape
key = key.view(num_tokens, -1, self.head_size)
key_rot = key[..., : self.rotary_dim]
key_pass = key[..., self.rotary_dim :]
key_rot = self.apply_rotary_emb.forward_native(
key_rot,
cos,
sin,
)
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
return query, key
def forward_cuda( # type: ignore[override]
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
return self.forward_native(positions, query, key)
def forward_xpu( # type: ignore[override]
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
# No fused XPU kernel for this 3D t/h/w rope; base
# MRotaryEmbedding.forward_xpu forwards an extra `offsets` arg that
# this class's forward_cuda override doesn't accept. Use native path.
return self.forward_native(positions, query, key)
@@ -1,153 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import async_tensor_h2d
# num_tokens/num_grids vary per request (incl. the ==1 class) and grids_ptr
# is an offset slice of the packed H2D buffer, so its 16B alignment varies
# with grid count; without these the kernel recompiles mid-serving.
@triton.jit(
do_not_specialize=["num_tokens", "num_grids"],
do_not_specialize_on_alignment=["grids_ptr"],
)
def _vit_mrope_setup_kernel(
inv_freq_t_ptr,
inv_freq_h_ptr,
inv_freq_w_ptr,
cos_out_ptr,
sin_out_ptr,
cu_seqlens_ptr, # (G + 1,) int32 token boundaries per grid
grids_ptr, # (G, 3) int32: t, h, w per grid
num_tokens,
num_grids,
half_t: tl.constexpr,
half_h: tl.constexpr,
half_w: tl.constexpr,
spatial_merge: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_D: tl.constexpr,
):
"""Build triton_mrope's (3, N, half_rot) cos/sin t/h/w planes in one
launch: compute each token's (t, h, w) position arithmetically and the
rotation in fp32 from the per-axis inv_freq. Only each axis's mrope
section columns are written; the rest is left uninitialized (the
consumer's per-axis masked loads read only the section columns)."""
pid = tl.program_id(0)
tok = pid * BLOCK_N + tl.arange(0, BLOCK_N)
tok_mask = tok < num_tokens
# Grid (segment) index per token; cu_seqlens is nondecreasing.
seg = tl.zeros([BLOCK_N], dtype=tl.int32)
for g in range(num_grids):
end = tl.load(cu_seqlens_ptr + g + 1)
seg += (tok >= end).to(tl.int32)
seg_start = tl.load(cu_seqlens_ptr + seg, mask=tok_mask, other=0)
grid_h = tl.load(grids_ptr + seg * 3 + 1, mask=tok_mask, other=1)
grid_w = tl.load(grids_ptr + seg * 3 + 2, mask=tok_mask, other=1)
local = tok - seg_start
rem = local % (grid_h * grid_w)
tpos = local // (grid_h * grid_w)
# Within a frame tokens are ordered by spatial-merge block (bh, bw, mh,
# mw), matching the reference ViT position-id permute.
bh = rem // (spatial_merge * grid_w)
r = rem % (spatial_merge * grid_w)
bw = r // (spatial_merge * spatial_merge)
r2 = r % (spatial_merge * spatial_merge)
hpos = bh * spatial_merge + r2 // spatial_merge
wpos = bw * spatial_merge + r2 % spatial_merge
d = tl.arange(0, BLOCK_D)
half_rot = half_t + half_h + half_w
row = (tok * half_rot)[:, None] + d[None, :]
plane = num_tokens * half_rot
# Per-axis compute+store to cap live registers; the fp32 trig sequence
# wants few elements per thread (hence small BLOCK_N, many warps).
in_t = d < half_t
inv_t = tl.load(inv_freq_t_ptr + d, mask=in_t, other=0.0)
freq = tpos.to(tl.float32)[:, None] * inv_t[None, :]
mask = tok_mask[:, None] & in_t[None, :]
tl.store(cos_out_ptr + row, tl.cos(freq), mask=mask)
tl.store(sin_out_ptr + row, tl.sin(freq), mask=mask)
in_h = (d >= half_t) & (d < half_t + half_h)
inv_h = tl.load(inv_freq_h_ptr + d - half_t, mask=in_h, other=0.0)
freq = hpos.to(tl.float32)[:, None] * inv_h[None, :]
mask = tok_mask[:, None] & in_h[None, :]
tl.store(cos_out_ptr + plane + row, tl.cos(freq), mask=mask)
tl.store(sin_out_ptr + plane + row, tl.sin(freq), mask=mask)
in_w = (d >= half_t + half_h) & (d < half_rot)
inv_w = tl.load(inv_freq_w_ptr + d - half_t - half_h, mask=in_w, other=0.0)
freq = wpos.to(tl.float32)[:, None] * inv_w[None, :]
mask = tok_mask[:, None] & in_w[None, :]
tl.store(cos_out_ptr + 2 * plane + row, tl.cos(freq), mask=mask)
tl.store(sin_out_ptr + 2 * plane + row, tl.sin(freq), mask=mask)
def vit_mrope_setup(
inv_freq_t: torch.Tensor,
inv_freq_h: torch.Tensor,
inv_freq_w: torch.Tensor,
grid_thw: list[list[int]],
spatial_merge_size: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""(3, total_N, half_rot_dim) fp32 cos/sin t/h/w planes for triton_mrope.
One fused kernel: positions are computed arithmetically from the grids
and the rotation comes from the per-axis inv_freq, so there is no
cos/sin cache to size or grow.
Args:
inv_freq_t: t-axis inverse frequencies, (t_dim // 2,).
inv_freq_h: h-axis inverse frequencies, (h_dim // 2,).
inv_freq_w: w-axis inverse frequencies, (w_dim // 2,).
grid_thw: per-grid [t, h, w] patch counts (host metadata).
spatial_merge_size: spatial merge factor of the ViT.
"""
device = inv_freq_t.device
cu = [0]
for t, h, w in grid_thw:
cu.append(cu[-1] + t * h * w)
n = cu[-1]
g = len(grid_thw)
# grid_thw is per-request host metadata, so one H2D is unavoidable;
# pack cu_seqlens and grids into a single pinned async copy.
buf = async_tensor_h2d(
cu + [d for grid in grid_thw for d in grid], device=device, dtype=torch.int32
)
cu_seqlens = buf[: g + 1]
grids = buf[g + 1 :].view(g, 3)
half_t = inv_freq_t.numel()
half_h = inv_freq_h.numel()
half_w = inv_freq_w.numel()
half_rot = half_t + half_h + half_w
cos = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
sin = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
_vit_mrope_setup_kernel[(triton.cdiv(n, 32),)](
inv_freq_t,
inv_freq_h,
inv_freq_w,
cos,
sin,
cu_seqlens,
grids,
n,
g,
half_t=half_t,
half_h=half_h,
half_w=half_w,
spatial_merge=spatial_merge_size,
BLOCK_N=32,
BLOCK_D=triton.next_power_of_2(half_rot),
num_warps=8,
)
return cos, sin
@@ -1,83 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""In-place interleaved RoPE on the Q/K slices of a packed QKV buffer."""
import torch
from vllm.triton_utils import tl, triton
# seqlen is not specialized: image sizes vary per request.
@triton.jit(do_not_specialize=["seqlen"])
def _packed_qk_rope_kernel(
qk_ptr, # (seqlen, 2 * nheads, headdim) strided view of the packed QKV
freqs_ptr, # (seqlen, rotary_dim) fp32 re/im view of the complex freqs
seqlen,
qk_row_stride,
qk_head_stride,
rotary_dim: tl.constexpr,
BLOCK_M: tl.constexpr,
):
BLOCK_K: tl.constexpr = triton.next_power_of_2(rotary_dim)
pid_m = tl.program_id(0)
pid_h = tl.program_id(1)
rm = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
rk = tl.arange(0, BLOCK_K)
mask_m = rm < seqlen
base = qk_ptr + pid_h * qk_head_stride + rm[:, None] * qk_row_stride
mask_x = mask_m[:, None] & (rk < rotary_dim)[None, :]
x = tl.load(base + rk[None, :], mask=mask_x, other=0.0).to(tl.float32)
f = tl.load(freqs_ptr + rm[:, None] * rotary_dim + rk[None, :], mask=mask_x)
cos, sin = tl.split(tl.reshape(f, (BLOCK_M, BLOCK_K // 2, 2)))
x0, x1 = tl.split(tl.reshape(x, (BLOCK_M, BLOCK_K // 2, 2)))
o0 = x0 * cos - x1 * sin
o1 = x1 * cos + x0 * sin
tl.store(base + rk[None, :], tl.interleave(o0, o1), mask=mask_x)
def packed_qk_rope_(xqkv: torch.Tensor, freqs_cis: torch.Tensor) -> None:
"""Rotate Q and K in place inside a packed QKV buffer.
Args:
xqkv: contiguous (seqlen, 3, nheads, headdim); only the Q and K
slices are rotated.
freqs_cis: contiguous (seqlen, headdim // 2) complex64 rotary freqs,
read through its interleaved re/im fp32 view.
Matches ``ApplyRotaryEmb(enable_fp32_compute=True)`` applied to Q and K
separately to within one ULP, but in one kernel with ~5x less memory
traffic. Both compute in fp32; the gap is which product the compiler
keeps exact inside the fused multiply-add, which differs per backend.
Requires triton: callers must check HAS_TRITON and use the unfused path
otherwise. Contract violations raise.
"""
assert xqkv.ndim == 4 and xqkv.size(1) == 3, xqkv.shape
assert xqkv.is_contiguous(), xqkv.shape
assert freqs_cis.ndim == 2 and freqs_cis.is_contiguous(), freqs_cis.shape
assert freqs_cis.dtype == torch.complex64, freqs_cis.dtype
seq_length, _, num_heads, head_dim = xqkv.shape
rotary_dim = freqs_cis.size(-1) * 2
assert freqs_cis.size(0) == seq_length, (freqs_cis.shape, xqkv.shape)
assert rotary_dim == head_dim, (freqs_cis.shape, xqkv.shape) # full rotation
qk = xqkv.as_strided(
(seq_length, 2 * num_heads, head_dim),
(3 * num_heads * head_dim, head_dim, 1),
)
BLOCK_M = 4
# seqlen on grid.x: HIP caps gridDim.y at 65535.
grid = (triton.cdiv(seq_length, BLOCK_M), 2 * num_heads)
_packed_qk_rope_kernel[grid](
qk,
freqs_cis.view(torch.float32),
seq_length,
qk.stride(0),
qk.stride(1),
rotary_dim=rotary_dim,
BLOCK_M=BLOCK_M,
num_warps=2 if rotary_dim <= 64 else 4,
)
@@ -23,7 +23,6 @@ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
from vllm.model_executor.models.transformers.base import VLLM_ATTN_ATTR
from vllm.model_executor.models.transformers.causal import CausalMixin
from vllm.model_executor.models.transformers.legacy import LegacyMixin
from vllm.model_executor.models.transformers.moe import MoEMixin
from vllm.model_executor.models.transformers.multimodal import (
MultiModalDummyInputsBuilder,
@@ -31,10 +30,6 @@ from vllm.model_executor.models.transformers.multimodal import (
MultiModalProcessingInfo,
MultiModalProcessor,
)
from vllm.model_executor.models.transformers.pooling import (
EmbeddingMixin,
SequenceClassificationMixin,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
if TYPE_CHECKING:
@@ -149,42 +144,6 @@ class TransformersMultiModalForCausalLM(MultiModalMixin, CausalMixin): ...
class TransformersMultiModalMoEForCausalLM(MoEMixin, MultiModalMixin, CausalMixin): ...
# Embedding models
class TransformersEmbeddingModel(EmbeddingMixin, LegacyMixin): ...
class TransformersMoEEmbeddingModel(EmbeddingMixin, MoEMixin): ...
@MULTIMODAL_REGISTRY.register_processor(
MultiModalProcessor,
info=MultiModalProcessingInfo,
dummy_inputs=MultiModalDummyInputsBuilder,
)
class TransformersMultiModalEmbeddingModel(EmbeddingMixin, MultiModalMixin): ...
# Sequence classification models
class TransformersForSequenceClassification(
SequenceClassificationMixin, LegacyMixin
): ...
class TransformersMoEForSequenceClassification(
SequenceClassificationMixin, MoEMixin
): ...
@MULTIMODAL_REGISTRY.register_processor(
MultiModalProcessor,
info=MultiModalProcessingInfo,
dummy_inputs=MultiModalDummyInputsBuilder,
)
class TransformersMultiModalForSequenceClassification(
SequenceClassificationMixin, MultiModalMixin
): ...
def __getattr__(name: str):
"""Handle imports of non-existent classes with a helpful error message."""
if name not in globals():
@@ -1,112 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transformers modeling backend mixins for pooling models."""
from typing import TYPE_CHECKING
import torch
import torch.nn as nn
from transformers import AutoModelForSequenceClassification
from vllm.config.utils import getattr_iter
from vllm.model_executor.layers.pooler import DispatchPooler
from vllm.model_executor.models.interfaces import SupportsCrossEncoding
from vllm.model_executor.models.interfaces_base import VllmModelForPooling
from .base import Base
if TYPE_CHECKING:
from vllm.config import VllmConfig
class ClassifierWithReshape(nn.Module):
"""Token extraction has already been applied in `pooler.pooling`.
Add dim to match expected input shape of `classifier.forward`.
"""
def forward(self, *args, **kwargs):
if len(args) > 0:
args = (args[0].unsqueeze(1), *args[1:])
return super().forward(*args, **kwargs)
class EmbeddingMixin(VllmModelForPooling, Base):
default_seq_pooling_type = "CLS"
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
# Skip VllmModelForPooling.__init__ and call the next class in MRO
super(VllmModelForPooling, self).__init__(
vllm_config=vllm_config, prefix=prefix
)
pooler_config = vllm_config.model_config.pooler_config
assert pooler_config is not None
self.pooler = DispatchPooler.for_embedding(pooler_config)
class SequenceClassificationMixin(SupportsCrossEncoding, VllmModelForPooling, Base):
default_seq_pooling_type = "CLS"
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
# Skip VllmModelForPooling.__init__ and call the next class in MRO
super(VllmModelForPooling, self).__init__(
vllm_config=vllm_config, prefix=prefix
)
pooler_config = vllm_config.model_config.pooler_config
assert pooler_config is not None
# Certain information about the model and classifier can only be
# inferred from the `ForSequenceClassification` class. Therefore, we
# instantiate it on the "meta" device to avoid allocating GPU memory.
with torch.device("meta"):
seq_cls_model = AutoModelForSequenceClassification.from_config(
self.config,
dtype=self.model_config.dtype,
trust_remote_code=self.model_config.trust_remote_code,
)
# When used for sequence classification, some models have their
# pooling layers removed. Make sure this is reflected in vLLM.
for module in seq_cls_model.modules():
if hasattr(module, "pooler") and module.pooler is None:
self.model.pooler = None
break
# Unlike `lm_head`, `classifier` is not always `nn.Linear`.
self.classifier = getattr_iter(seq_cls_model, ["classifier", "score"], None)
if self.classifier is None:
raise ValueError(
"Could not find `classifier` or `score` layer in the "
"`AutoModelForSequenceClassification` instance."
)
self.init_parameters(self.classifier, dtype=self.model_config.head_dtype)
# Order `ClassifierWithReshape` ahead of the classifier's own class so that
# its `super().forward(...)` reaches the original implementation.
self.classifier.__class__ = type(
"ClassifierWithReshape",
(ClassifierWithReshape, type(self.classifier)),
{},
)
self.pooler = DispatchPooler.for_seq_cls(
pooler_config,
classifier=self.classifier,
)