[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,602 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import cutlass
import cutlass.cute as cute
import torch
from cuda.bindings.driver import CUstream
from cutlass import Float32, Int32, Uint32, Uint64
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import recast_val
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
CuTeDSLLaunchSpec,
VllmCuTeDSLJitKernel,
)
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
)
from vllm.triton_utils import tl, triton
def _dcp_merge_topk_and_interleave(vllm_config: Any) -> tuple[int, int]:
"""(topk, cp_interleave) the DCP merge runs with.
Kpool indexers (GLM-5.x) merge pool ids: topk and the interleave are both
in units of ``index_kpool`` tokens.
"""
hf_text_config = vllm_config.model_config.hf_text_config
topk = hf_text_config.index_topk
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
kpool = getattr(hf_text_config, "index_kpool", None) or 1
if kpool > 1:
topk //= kpool
cp_interleave //= kpool
return topk, cp_interleave
def stable_topk_from_gathered_candidates_cutedsl(
gathered: torch.Tensor,
topk: int,
out: torch.Tensor | None = None,
) -> torch.Tensor:
if out is None:
out = torch.empty(
(gathered.shape[0], topk),
dtype=torch.int32,
device=gathered.device,
)
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL(gathered, out, topk=topk)
return out
def pack_dcp_topk_candidates_cutedsl(
logits: torch.Tensor,
topk_indices: torch.Tensor,
packed: torch.Tensor,
dcp_rank: int,
dcp_world_size: int,
cp_interleave: int,
row_starts: torch.Tensor | None,
) -> None:
topk = topk_indices.shape[1]
row_starts_arg = row_starts if row_starts is not None else topk_indices
_PACK_DCP_TOPK_CANDIDATES_KERNEL(
logits,
topk_indices,
packed,
row_starts_arg,
logits_stride0=logits.stride(0),
logits_stride1=logits.stride(1),
topk_stride0=topk_indices.stride(0),
topk_stride1=topk_indices.stride(1),
packed_stride0=packed.stride(0),
packed_stride1=packed.stride(1),
packed_stride2=packed.stride(2),
num_cols=logits.shape[1],
dcp_rank=dcp_rank,
dcp_world_size=dcp_world_size,
cp_interleave=cp_interleave,
has_row_starts=row_starts is not None,
topk=topk,
block_size=512,
)
class PackDCPTopkCandidatesKernel(
VllmTritonJitKernel["PackDCPTopkCandidatesKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
dcp_rank: int
dcp_world_size: int
cp_interleave: int
has_row_starts: bool
topk: int
block_size: int
# These scalars only describe runtime layouts and bounds; the constexpr
# fields below own the launch geometry and algorithmic specialization.
@staticmethod
@triton.jit(
do_not_specialize=[
"logits_stride0",
"logits_stride1",
"topk_stride0",
"topk_stride1",
"packed_stride0",
"packed_stride1",
"packed_stride2",
"num_cols",
]
)
def kernel(
logits,
topk_indices,
packed,
row_starts,
logits_stride0,
logits_stride1,
topk_stride0,
topk_stride1,
packed_stride0,
packed_stride1,
packed_stride2,
num_cols,
DCP_RANK: tl.constexpr,
DCP_WORLD_SIZE: tl.constexpr,
CP_INTERLEAVE: tl.constexpr,
HAS_ROW_STARTS: tl.constexpr,
TOPK: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0)
tile = tl.program_id(1)
cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = cols < TOPK
local_idx = tl.load(
topk_indices + row * topk_stride0 + cols * topk_stride1,
mask=mask,
other=-1,
)
valid = local_idx >= 0
safe_local_idx = tl.maximum(local_idx, 0)
row_start = 0
if HAS_ROW_STARTS:
row_start = tl.load(row_starts + row)
score_col = safe_local_idx + row_start
score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0))
score = tl.load(
logits + row * logits_stride0 + score_col * logits_stride1,
mask=mask & valid,
other=-float("inf"),
)
global_id = (
(safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE)
+ DCP_RANK * CP_INTERLEAVE
+ safe_local_idx % CP_INTERLEAVE
)
global_id = tl.where(valid, global_id, -1)
packed_base = packed + row * packed_stride0 + cols * packed_stride1
tl.store(packed_base, score, mask=mask)
tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask)
def dispatch( # type: ignore[override]
self,
*,
has_row_starts: bool,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(
**compile_key_fields,
has_row_starts=has_row_starts,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
if dcp_world_size <= 1:
return []
topk, cp_interleave = _dcp_merge_topk_and_interleave(vllm_config)
if topk <= 0:
return []
try:
from vllm.distributed.parallel_state import get_dcp_group
dcp_rank = get_dcp_group().rank_in_group
except Exception:
dcp_rank = 0
return self._trace_dispatch(self.dispatch)(
dcp_rank=dcp_rank,
dcp_world_size=dcp_world_size,
cp_interleave=cp_interleave,
has_row_starts=(False, True),
topk=topk,
block_size=512,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
fp32_ptr = TritonWarmupTensor(torch.float32)
int32_ptr = TritonWarmupTensor(torch.int32)
return dict(
logits=fp32_ptr,
topk_indices=TritonWarmupTensor(torch.int32, shape=(1, compile_key.topk)),
packed=fp32_ptr,
row_starts_arg=int32_ptr,
logits_stride0=1,
logits_stride1=1,
topk_stride0=1,
topk_stride1=1,
packed_stride0=1,
packed_stride1=1,
packed_stride2=1,
num_cols=1,
dcp_rank=compile_key.dcp_rank,
dcp_world_size=compile_key.dcp_world_size,
cp_interleave=compile_key.cp_interleave,
has_row_starts=compile_key.has_row_starts,
topk=compile_key.topk,
block_size=compile_key.block_size,
)
@kernel_launcher
def __call__(
self,
logits: torch.Tensor,
topk_indices: torch.Tensor,
packed: torch.Tensor,
row_starts_arg: torch.Tensor,
*,
logits_stride0: int,
logits_stride1: int,
topk_stride0: int,
topk_stride1: int,
packed_stride0: int,
packed_stride1: int,
packed_stride2: int,
num_cols: int,
dcp_rank: int,
dcp_world_size: int,
cp_interleave: int,
has_row_starts: bool,
topk: int,
block_size: int,
) -> LaunchSpec:
grid = (topk_indices.shape[0], triton.cdiv(topk, block_size))
return grid, dict(
row_starts=row_starts_arg,
DCP_RANK=dcp_rank,
DCP_WORLD_SIZE=dcp_world_size,
CP_INTERLEAVE=cp_interleave,
HAS_ROW_STARTS=has_row_starts,
TOPK=topk,
BLOCK_SIZE=block_size,
num_warps=8,
)
class StableTopKFromGatheredCandidatesKernel(
VllmCuTeDSLJitKernel["StableTopKFromGatheredCandidatesKernel.CompileKey"]
):
tb_size = 512
hist_bins = 2048
radix_bits = (hist_bins - 1).bit_length()
key_bits = Uint64.width
radix_passes = (key_bits + radix_bits - 1) // radix_bits
final_radix_bits = key_bits - radix_bits * (radix_passes - 1)
hist_chunks = (hist_bins + tb_size - 1) // tb_size
warps_per_block = tb_size // cute.arch.WARP_SIZE
@dataclass(frozen=True)
class CompileKey:
topk: int
num_candidates: int
@staticmethod
def kernel(compile_key: CompileKey) -> Any:
tb_size = StableTopKFromGatheredCandidatesKernel.tb_size
hist_bins = StableTopKFromGatheredCandidatesKernel.hist_bins
radix_bits = StableTopKFromGatheredCandidatesKernel.radix_bits
key_bits = StableTopKFromGatheredCandidatesKernel.key_bits
radix_passes = StableTopKFromGatheredCandidatesKernel.radix_passes
final_radix_bits = StableTopKFromGatheredCandidatesKernel.final_radix_bits
hist_chunks = StableTopKFromGatheredCandidatesKernel.hist_chunks
warps_per_block = StableTopKFromGatheredCandidatesKernel.warps_per_block
topk = compile_key.topk
assert hist_bins == 1 << radix_bits
assert compile_key.num_candidates % tb_size == 0, (
"StableTopKFromGatheredCandidatesKernel requires candidate count "
f"to be a multiple of {tb_size}, got {compile_key.num_candidates}"
)
keys_per_thread = compile_key.num_candidates // tb_size
@cute.struct
class SharedStorage:
hist: cute.struct.MemRange[Int32, hist_bins]
committed_count: cute.struct.MemRange[Int32, 1]
running_count: cute.struct.MemRange[Int32, 1]
threshold_bin: cute.struct.MemRange[Int32, 1]
threshold_found: cute.struct.MemRange[Int32, 1]
include_threshold_bin: cute.struct.MemRange[Int32, 1]
prefix_s: cute.struct.Align[cute.struct.MemRange[Uint64, 1], 8]
warp_totals: cute.struct.MemRange[Int32, warps_per_block]
shared_storage = SharedStorage
@cute.jit
def warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32:
for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1):
offset = 1 << i
partial = cute.arch.shuffle_sync_up(
val, offset=offset, mask_and_clamp=0
)
if lane >= offset:
val += partial
return val
@cute.jit
def block_scan_inclusive_i32(
val: Int32,
lane: Int32,
warp_id: Int32,
warp_scratch: cute.Tensor,
warps_per_block: int,
) -> Int32:
prefix = warp_scan_inclusive_i32(val, lane)
if lane == Int32(cute.arch.WARP_SIZE - 1):
warp_scratch[0, warp_id] = prefix
cute.arch.sync_threads()
if warp_id == Int32(0):
warp_total = Int32(0)
if lane < Int32(warps_per_block):
warp_total = warp_scratch[0, lane]
warp_prefix = warp_scan_inclusive_i32(warp_total, lane)
if lane < Int32(warps_per_block):
warp_scratch[0, lane] = warp_prefix - warp_total
cute.arch.sync_threads()
return prefix + warp_scratch[0, warp_id]
@cute.jit
def stable_key(score: Float32, token_id: Int32) -> Uint64:
bits = recast_val(score, Uint32)
mask = Uint32(0x80000000)
if (bits & Uint32(0x80000000)) != Uint32(0):
mask = Uint32(0xFFFFFFFF)
score_key = Uint64(bits ^ mask) << Uint64(32)
id_key = Uint64(~Uint32(token_id))
key = score_key | id_key
if token_id < Int32(0):
key = Uint64(0)
return key
@cute.jit
def prefix_matches(
key: Uint64,
prefix: Uint64,
prefix_bits: Int32,
):
matches = prefix_bits == Int32(0)
if prefix_bits != Int32(0):
shift = Int32(key_bits) - prefix_bits
matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift))
return matches
@cute.jit
def radix_pass(
keys: cute.Tensor,
output: cute.Tensor,
storage,
tid: Int32,
step: Int32,
bits: int,
is_final_pass: bool,
):
hist_smem = storage.hist.get_tensor(cute.make_layout((hist_bins,)))
committed_count_smem = storage.committed_count.data_ptr()
running_count_smem = storage.running_count.data_ptr()
threshold_bin_smem = storage.threshold_bin.data_ptr()
threshold_found_smem = storage.threshold_found.data_ptr()
include_threshold_bin_smem = storage.include_threshold_bin.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
warp_totals_smem = storage.warp_totals.get_tensor(
cute.make_layout((1, warps_per_block))
)
prefix_bits = step * Int32(radix_bits)
num_bins = 1 << bits
block_scan_iterations = (num_bins + tb_size - 1) // tb_size
shift = Int32(key_bits) - prefix_bits - Int32(bits)
bin_mask = Uint64(num_bins - 1)
prefix = prefix_smem.load()
for chunk in cutlass.range_constexpr(hist_chunks):
hist_smem[tid + Int32(chunk * tb_size)] = Int32(0)
if tid == Int32(0):
running_count_smem.store(committed_count_smem.load())
include_threshold_bin_smem.store(Int32(0))
threshold_found_smem.store(Int32(0))
cute.arch.sync_threads()
for key_idx in cutlass.range_constexpr(keys_per_thread):
key = keys[key_idx]
if prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
cute.arch.atomic_add(
hist_smem.iterator + bin_idx,
Int32(1),
sem="relaxed",
scope="cta",
)
cute.arch.sync_threads()
lane = cute.arch.lane_idx()
warp_id = cute.arch.warp_idx()
# Each iteration scans one tb_size-wide slice of bins, high to low.
iter = Int32(0)
threshold_found = threshold_found_smem.load()
while threshold_found == Int32(0) and iter < Int32(block_scan_iterations):
bin_idx = Int32(num_bins - 1) - (iter * Int32(tb_size) + tid)
count = hist_smem[bin_idx]
chunk_inclusive = block_scan_inclusive_i32(
count,
lane,
warp_id,
warp_totals_smem,
warps_per_block,
)
running_count = running_count_smem.load()
prior_in_scan_slice = chunk_inclusive - count
remaining = Int32(topk) - running_count - prior_in_scan_slice
if count > Int32(0) and remaining > Int32(0) and remaining <= count:
threshold_bin_smem.store(bin_idx)
if count <= remaining or cutlass.const_expr(is_final_pass):
include_threshold_bin_smem.store(Int32(1))
threshold_found_smem.store(Int32(1))
# Barrier: every thread must finish reading running_count for this
# slice before tb_size-1 advances it, else a warp racing ahead to
# the store makes a lagging thread double-count the slice total
# (-> remaining too small -> threshold too high -> under-fill).
cute.arch.sync_threads()
if tid == Int32(tb_size - 1):
running_count_smem.store(running_count + chunk_inclusive)
cute.arch.sync_threads()
threshold_found = threshold_found_smem.load()
iter += Int32(1)
threshold = threshold_bin_smem.load()
should_include_threshold = include_threshold_bin_smem.load() != Int32(0)
for key_idx in cutlass.range_constexpr(keys_per_thread):
key = keys[key_idx]
if prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
selected = bin_idx > threshold
if should_include_threshold:
selected = selected or bin_idx == threshold
if selected:
dst = cute.arch.atomic_add(
committed_count_smem,
Int32(1),
sem="relaxed",
scope="cta",
)
if dst < Int32(topk):
output[dst] = recast_val(~Uint32(key), Int32)
cute.arch.sync_threads()
pass_finished = include_threshold_bin_smem.load()
if tid == Int32(0) and pass_finished == Int32(0):
prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift)))
cute.arch.sync_threads()
return pass_finished
@cute.kernel
def device_kernel(input: cute.Tensor, out: cute.Tensor):
row, _, _ = cute.arch.block_idx()
tid, _, _ = cute.arch.thread_idx()
input_row = input[row, None, None]
output_row = out[row, None]
keys = cute.make_rmem_tensor((keys_per_thread,), Uint64)
smem = cutlass.utils.SmemAllocator()
storage = smem.allocate(shared_storage, 8)
committed_count_smem = storage.committed_count.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
for i in range(tid, topk, tb_size):
output_row[i] = Int32(-1)
for key_idx in cutlass.range_constexpr(keys_per_thread):
col = tid + Int32(key_idx * tb_size)
score = Float32(input_row[col, 0])
token_id = Int32(input_row[col, 1])
keys[key_idx] = stable_key(score, token_id)
if tid == Int32(0):
committed_count_smem.store(Int32(0))
prefix_smem.store(Uint64(0))
cute.arch.sync_threads()
step = Int32(0)
finished = Int32(0)
while finished == Int32(0) and step < Int32(radix_passes - 1):
finished = radix_pass(
keys,
output_row,
storage,
tid,
step,
radix_bits,
False,
)
step += Int32(1)
if finished == Int32(0):
radix_pass(
keys,
output_row,
storage,
tid,
Int32(radix_passes - 1),
final_radix_bits,
True,
)
@cute.jit
def host_entrypoint(
gathered: cute.Tensor,
out: cute.Tensor,
stream: CUstream,
):
grid = (gathered.shape[0], 1, 1)
device_kernel(gathered, out).launch(
grid=grid,
block=(tb_size, 1, 1),
stream=stream,
)
return host_entrypoint
def dispatch( # type: ignore[override]
self,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(**compile_key_fields)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
if dcp_world_size <= 1:
return []
topk, _ = _dcp_merge_topk_and_interleave(vllm_config)
if topk <= 0:
return []
return self._trace_dispatch(self.dispatch)(
topk=topk,
num_candidates=topk * dcp_world_size,
)
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
num_rows = cute.sym_int()
gathered = cute.runtime.make_fake_tensor(
Float32,
(num_rows, compile_key.num_candidates, 2),
stride=(cute.sym_int64(divisibility=2), 2, 1),
assumed_align=8,
)
out = make_fake_tensor(
Int32,
(num_rows, compile_key.topk),
divisibility=1,
)
return gathered, out
@kernel_launcher
def __call__(
self,
gathered: torch.Tensor,
out: torch.Tensor,
*,
topk: int,
) -> CuTeDSLLaunchSpec[CompileKey]:
compile_key = self.dispatch(
topk=topk,
num_candidates=gathered.shape[1],
)
return compile_key, (gathered, out)
_PACK_DCP_TOPK_CANDIDATES_KERNEL = PackDCPTopkCandidatesKernel()
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL = StableTopKFromGatheredCandidatesKernel()
@@ -59,10 +59,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
TritonW4A16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
XPUW4A8IntLinearKernel,
XPUwNa16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
ZentorchWNA16LinearKernel,
)
@@ -91,9 +87,6 @@ from vllm.model_executor.kernels.linear.mxfp4.marlin import (
from vllm.model_executor.kernels.linear.mxfp4.pytorch import (
TorchMxFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp4.xpu import (
XPUMxFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp6 import (
MxFp6LinearKernel,
MxFp6LinearLayerConfig,
@@ -132,9 +125,6 @@ from vllm.model_executor.kernels.linear.mxfp8.pytorch import (
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.xpu import (
XPUMxFp8LinearKernel,
)
from vllm.model_executor.kernels.linear.nvfp4 import (
NvFp4LinearKernel,
NvFp4LinearLayerConfig,
@@ -228,11 +218,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import (
TritonFP8ScaledMMLinearKernel,
TritonInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
XPUFp8BlockScaledMMKernel,
XPUW8A8FP8LinearKernel,
XPUW8A16FP8LinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
ZentorchInt8ScaledMMLinearKernel,
)
@@ -373,13 +358,6 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
EmulationMxfp6LinearKernel,
EmulationMxfp4LinearKernel,
},
"xpu": {
XPUW8A8FP8LinearKernel,
XPUFp8BlockScaledMMKernel,
},
"xpu_woq": {
XPUW8A16FP8LinearKernel,
},
}
@@ -444,7 +422,6 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]
HummingInt8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [AiterInt8ScaledMMLinearKernel, TritonInt8ScaledMMLinearKernel],
PlatformEnum.XPU: [TritonInt8ScaledMMLinearKernel],
}
# in priority/performance order (when available)
@@ -473,12 +450,6 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
PerTensorTorchFP8ScaledMMLinearKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
],
PlatformEnum.XPU: [
XPUW8A16FP8LinearKernel,
XPUW8A8FP8LinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
],
}
@@ -504,11 +475,6 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
CPUFp8W8A8BlockScaledMMKernel, # W8A8 preferred; falls back to W8A16 below
CPUFp8BlockScaledMMKernel,
],
PlatformEnum.XPU: [
XPUFp8BlockScaledMMKernel,
TritonFp8BlockScaledMMKernel,
BlockWiseTorchFP8ScaledMMLinearKernel,
],
}
_POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
@@ -522,9 +488,6 @@ _POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]
PlatformEnum.CPU: [
# To be added
],
PlatformEnum.XPU: [
XPUW8A16FP8LinearKernel,
],
}
# in priority/performance order (when available)
@@ -545,10 +508,6 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
ConchLinearKernel,
ExllamaLinearKernel,
],
PlatformEnum.XPU: [
XPUW4A8IntLinearKernel,
XPUwNa16LinearKernel,
],
PlatformEnum.CPU: [
Dynamic4bitLinearKernel,
ZentorchWNA16LinearKernel,
@@ -574,11 +533,6 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
RocmDotScaledMxfp8LinearKernel,
EmulationMxfp8LinearKernel,
],
PlatformEnum.XPU: [
XPUMxFp8LinearKernel,
TorchMxFp8LinearKernel,
EmulationMxfp8LinearKernel,
],
}
_POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
@@ -626,10 +580,6 @@ _POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = {
AiterMxfp4LinearKernel,
EmulationMxfp4LinearKernel,
],
PlatformEnum.XPU: [
XPUMxFp4LinearKernel,
TorchMxFp4LinearKernel,
],
}
# TODO make all kernels inherit from MMLinearKernel
@@ -1348,7 +1298,6 @@ __all__ = [
"MacheteLinearKernel",
"MarlinLinearKernel",
"TritonW4A16LinearKernel",
"XPUW4A8IntLinearKernel",
"XPUwNa16LinearKernel",
"init_mxfp8_linear_kernel",
"Mxfp8LinearKernel",
@@ -1373,7 +1322,6 @@ __all__ = [
"FlashInferCutlassMxfp8LinearKernel",
"FlashInferTrtllmMxfp8LinearKernel",
"MarlinMxfp8LinearKernel",
"XPUMxFp8LinearKernel",
"TorchMxFp8LinearKernel",
"EmulationMxfp8LinearKernel",
"CutlassNvFp4LinearKernel",
@@ -38,10 +38,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
TritonW4A16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
XPUW4A8IntLinearKernel,
XPUwNa16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
ZentorchWNA16LinearKernel,
)
@@ -60,7 +56,6 @@ __all__ = [
"RDNA3W4A16LinearKernel",
"RDNAHybridW4A16LinearKernel",
"TritonW4A16LinearKernel",
"XPUW4A8IntLinearKernel",
"XPUwNa16LinearKernel",
"ZentorchWNA16LinearKernel",
]
@@ -1,218 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch.nn.parameter import Parameter
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
_XPUWNA16_SUPPORTED_QUANT_TYPES = (scalar_types.uint4, scalar_types.uint4b8)
logger = init_logger(__name__)
class XPUwNa16LinearKernel(MPLinearKernel):
@classmethod
def get_min_capability(cls) -> int:
return -1
@classmethod
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUwNa16 only supported on XPU"
if c.act_type != torch.bfloat16 and c.act_type != torch.float16:
return False, "XPUwNa16 only supports BF16/FP16 activations"
if c.weight_type not in _XPUWNA16_SUPPORTED_QUANT_TYPES:
return (
False,
f"Quant type ({c.weight_type}) not supported by "
"XPUwNa16, supported types are: "
f"{_XPUWNA16_SUPPORTED_QUANT_TYPES}",
)
if c.group_size != -1 and c.group_size % 32 != 0:
return (
False,
f"Group size ({c.group_size}) not supported by "
"XPUwNa16, supported group sizes are multiples of 32",
)
if c.partition_weight_shape[0] % 32 != 0:
return (
False,
f"Input size ({c.partition_weight_shape[0]}) not supported by "
"XPUwNa16, supported sizes are multiples of 32",
)
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module):
# Default name since marlin requires empty parameter for zp,
# TODO: remove this requirement from marlin (allow optional tensors)
if self.w_zp_name is None:
self.w_zp_name = "w_zp"
need_transpose = False
qweight_shape = getattr(layer, self.w_q_name).shape
scale_shape = getattr(layer, self.w_s_name).shape
# gptq marlin and compressed tensors wna16 expect different default
# layouts for weight and scale, so we check the shapes to determine
# if we need to transpose
if qweight_shape[0] != scale_shape[0]:
need_transpose = True
if need_transpose:
getattr(layer, self.w_q_name).data = (
getattr(layer, self.w_q_name).data.t().contiguous()
)
getattr(layer, self.w_s_name).data = getattr(layer, self.w_s_name).data
else:
getattr(layer, self.w_s_name).data = (
getattr(layer, self.w_s_name).data.t().contiguous()
)
if self.config.zero_points:
# (FIXME): maybe zero points should also be transposed.
getattr(layer, self.w_zp_name).data = (
getattr(layer, self.w_zp_name).data.t().contiguous()
)
else:
weight_zero_point = torch.Tensor([8]).to(torch.int8).to("xpu")
setattr(
layer, self.w_zp_name, Parameter(weight_zero_point, requires_grad=False)
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
reshaped_x = x.reshape(-1, x.shape[-1])
w_q, w_s, w_zp = self._get_weight_params(layer)
out = torch.ops._xpu_C.int4_gemm_w4a16(
reshaped_x,
w_q.t(),
bias if bias is not None else None,
w_s,
w_zp,
self.config.group_size,
None, # Retained by the external XPU op ABI.
)
return out
class XPUW4A8IntLinearKernel(MPLinearKernel):
"""XPU kernel for W4A8 integer quantization using oneDNN int4_gemm_w4a8.
Weights are symmetric group-quantized int4 packed as uint4.
Activations are dynamically quantized per-token to symmetric int8.
"""
@classmethod
def get_min_capability(cls) -> int:
return -1
@classmethod
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUW4A8Int only supported on XPU"
if c.act_type not in (torch.bfloat16, torch.float16):
return False, "XPUW4A8Int requires BF16/FP16 activations"
if c.weight_type != scalar_types.int4:
return (
False,
f"XPUW4A8Int requires int4 weights, got {c.weight_type}",
)
if c.zero_points:
return False, "XPUW4A8Int only supports symmetric weight quantization"
if c.group_size != -1 and c.group_size % 32 != 0:
return (
False,
f"Group size ({c.group_size}) not supported by XPUW4A8Int, "
"must be a multiple of 32",
)
in_size, out_size = c.partition_weight_shape
if in_size % 8 != 0 or out_size % 8 != 0:
return (
False,
f"in/out sizes ({in_size}, {out_size}) must be multiples of 8",
)
if c.act_type != torch.float16:
logger.warning_once(
"XPUW4A8IntLinearKernel is running with model dtype %s, "
"but int4_gemm_w4a8 produces float16 output. Recommend "
"setting --dtype float16 for best performance.",
c.act_type,
)
return True, None
def _pack_int4_weight(self, w: torch.Tensor) -> torch.Tensor:
# w is [N, K] int8 with values in [-8, 7]
w_u4 = w.to(torch.int32) + 8 # shift to [0, 15]
w_u4 = w_u4.reshape(w.shape[0], w.shape[1] // 8, 8) # [N, K/8, 8]
shifts = torch.arange(0, 32, 4, dtype=torch.int32, device=w.device)
packed = ((w_u4 & 0xF) << shifts[None, None, :]).sum(dim=2).to(torch.int32)
return packed
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight_scale.data = layer.weight_scale.data.t().contiguous()
device = layer.weight_packed.device
# TODO: support asymmetric quantization
weight_zero_point = torch.tensor([8], dtype=torch.int8, device=device)
layer.weight_zero_point = Parameter(weight_zero_point, requires_grad=False)
# weight_packed is [out, in] int8, signed int4 values in [-8, 7]
w = layer.weight_packed.data # [out, in]
# TODO: implement asym case
packed = self._pack_int4_weight(w) # [out, in/8] packed uint4
replace_parameter(
layer,
self.w_q_name,
torch.nn.Parameter(packed, requires_grad=False),
)
# Free the original unpacked int8 weight (still registered as "weight")
# to avoid double-storing both int8 [N, K] and int32 [N, K/8] in memory.
delattr(layer, "weight")
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
reshaped_x = x.reshape(-1, x.shape[-1]) # [M, K]
from vllm._xpu_ops import xpu_ops as ops
# TODO: static and asymmetric quantization case
# Common code for CompressedTensorsW4A8Int does not read act symmetry data
quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref(
reshaped_x, True, 8
)
out = torch.ops._xpu_C.int4_gemm_w4a8(
quant_x,
x_scale,
x_zero,
layer.weight_packed.t(),
layer.weight_scale,
layer.weight_zero_point,
self.config.group_size,
None, # Retained by the external XPU op ABI.
bias,
)
return out.to(x.dtype)
@@ -1,58 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
xpu_mxfp4_quantize as quant_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
class XPUMxFp4LinearKernel(MxFp4LinearKernel):
"""MXFP4 W4A4 GEMM on XPU."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUMxFp4 only support on XPU"
return True, None
@classmethod
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
if config.activation_quant_key != kMxfp4Dynamic:
return False, "only supports MXFP4 dynamic activation"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
weight = layer.weight.view(torch.float4_e2m1fn_x2)
replace_parameter(layer, "weight", weight.data.t())
weight_scale = layer.weight_scale.view(torch.float8_e8m0fnu)
weight_scale = weight_scale.t().contiguous()
replace_parameter(layer, "weight_scale", weight_scale.data)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
out_dtype = x.dtype
x_fp4, x_blockscale = quant_mxfp4(x)
return torch.ops._xpu_C.fp4_gemm(
x_fp4,
layer.weight,
x_blockscale,
layer.weight_scale,
out_dtype,
bias,
)
@@ -1,82 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
xpu_mxfp8_quantize as quant_mxfp8,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
class XPUMxFp8LinearKernel(Mxfp8LinearKernel):
"""MXFP8 W8A8 GEMM on XPU."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUMxFp8 only support on XPU"
return True, None
@classmethod
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Checkpoint scale is [N, K//32] (one E8M0 scale per 32 elements).
# oneDNN fp8_gemm requires contiguous [K//32, N] layout. Store the
# transposed contiguous buffer as a .t() view so that:
# - dequantization consumers still see the checkpoint shape
# - apply_weights recovers the oneDNN layout via .t() at zero cost
weight_scale = layer.weight_scale.view(torch.float8_e8m0fnu)
scale_kn = weight_scale.data.t().contiguous()
replace_parameter(layer, "weight_scale", scale_kn.t())
if getattr(layer, "is_bmm", False):
self._prepare_bmm_params(layer, scale_kn)
def _prepare_bmm_params(
self, layer: torch.nn.Module, scale_kn: torch.Tensor
) -> None:
"""Precompute batched weight and scale for grouped fp8_bmm (e.g. wo_a).
Splits scale [K//32, N_total] into [G, K//32, N_per_group] and weight
[N_total, K] into contiguous [G, K, N_per_group] for batch GEMM.
"""
batch = layer.bmm_batch_size
k_blocks, n_blocks = scale_kn.shape
layer.bmm_scale = (
scale_kn.reshape(k_blocks, batch, n_blocks // batch)
.permute(1, 0, 2)
.contiguous()
)
w = layer.weight
n_total, k = w.shape
layer.bmm_weight = (
w.reshape(batch, n_total // batch, k).permute(0, 2, 1).contiguous()
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
out_dtype = x.dtype
x_fp8, x_scale = quant_mxfp8(x)
# Weight is [N, K]; .t() gives a [K, N] view without copying.
# Scale is stored as [N, K//32]; .t() recovers the contiguous
# [K//32, N] buffer that oneDNN expects.
return torch.ops._xpu_C.fp8_gemm(
x_fp8,
layer.weight.t(),
out_dtype,
x_scale,
layer.weight_scale.t(),
bias,
)
@@ -42,9 +42,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import (
TritonFP8ScaledMMLinearKernel,
TritonInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
XPUFp8BlockScaledMMKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
ZentorchInt8ScaledMMLinearKernel,
)
@@ -73,5 +70,4 @@ __all__ = [
"CPUFp8BlockScaledMMKernel",
"CPUFp8PerTensorScaledMMLinearKernel",
"CPUFP8W8A8ScaledMMLinearKernel",
"XPUFp8BlockScaledMMKernel",
]
@@ -1,284 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Sequence
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel
from .ScaledMMLinearKernel import FP8ScaledMMLinearKernel, FP8ScaledMMLinearLayerConfig
class XPUW8A8FP8LinearKernel(FP8ScaledMMLinearKernel):
_SUPPORTED_ACT_QUANT_KEYS = {
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
}
_SUPPORTED_WEIGHT_QUANT_KEYS = {
kFp8StaticChannelSym,
kFp8StaticTensorSym,
}
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUW8A8FP8Linear only support on XPU"
return True, None
@classmethod
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
if c.weight_quant_key not in cls._SUPPORTED_WEIGHT_QUANT_KEYS:
return (
False,
"XPUW8A8FP8Linear only support per-channel and per-tensor quantization",
)
if c.activation_quant_key not in cls._SUPPORTED_ACT_QUANT_KEYS:
return (
False,
"XPUW8A8FP8Linear only support per-tensor and per-token activation "
"quantization",
)
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
return False, "XPUW8A8FP8Linear only support FP8 weight dtype"
if c.activation_quant_key.dtype not in {
torch.float8_e5m2,
torch.float8_e4m3fn,
}:
return False, "XPUW8A8FP8Linear only support FP8 activation dtype"
return True, None
def __init__(
self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str]
) -> None:
super().__init__(c, layer_param_names)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Ensure weight is stored as C-contiguous [K, N] (KN layout).
Checkpoints store weight as [N, K]; fp8_gemm requires [K, N],
C-contiguous. Three incoming layouts are possible:
• [N, K] C-contiguous ← direct checkpoint → .t().contiguous()
• [K, N] Fortran-order ← fp8.py's weight.t() → .contiguous()
• [K, N] C-contiguous ← already correct → no-op
For square weights (K == N) the shape is ambiguous; contiguity is used
as a proxy: C-contiguous ≡ checkpoint [N, K] (needs transpose);
Fortran-order ≡ fp8.py already transposed (needs only contiguous).
"""
K = getattr(layer, "input_size_per_partition", self.config.weight_shape[1])
N = getattr(layer, "output_size_per_partition", self.config.weight_shape[0])
w = layer.weight
if w.shape not in {(K, N), (N, K)}:
raise ValueError(
f"XPUFP8ScaledMM expects weight shape ({K},{N}) or ({N},{K}), "
f"but got {tuple(w.shape)}"
)
needs_transpose = w.shape == (N, K) if K != N else w.is_contiguous()
layer_weight = w.t() if needs_transpose else w
replace_parameter(layer, "weight", layer_weight)
ws = layer.weight_scale
if ws.numel() == 1:
replace_parameter(layer, "weight_scale", ws.reshape(1))
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
# B is C-contiguous [K, N] from process_weights_after_loading.
# fp8_gemm routes on scale dtype (float32) and numel:
# As [1] → per-tensor (numel==1 branch)
# As [M,1] → per-token (group={1,K} branch, broadcast across K)
# Bs [1] → per-tensor
# Bs [N] → per-channel (mask=bit1 branch)
# No shape manipulation needed here.
output = torch.ops._xpu_C.fp8_gemm(A, B, out_dtype, As, Bs, bias)
return output.view(*output_shape)
class XPUW8A16FP8LinearKernel(FP8ScaledMMLinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUW8A16FP8Linear only support on XPU"
return True, None
@classmethod
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
if c.weight_quant_key not in {kFp8StaticChannelSym, kFp8StaticTensorSym}:
return (
False,
"XPUW8A16FP8Linear only support per-channel and per-tensor "
"quantization",
)
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
return False, "XPUW8A16FP8Linear only support FP8 weight dtype"
return True, None
def __init__(
self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str]
) -> None:
assert self.can_implement(c)[0]
assert self.is_supported()[0]
self.config = c
self.layer_param_names = layer_param_names
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# fp8_gemm_w8a16 expects weight in [in, out] layout.
# Transpose if weight is still in [out, in] layout.
# For square matrices, use contiguity as tie-breaker:
# checkpoint weights are contiguous, .t() views are not.
weight = layer.weight
out_features, in_features = self.config.weight_shape
if weight.shape == (out_features, in_features) and (
in_features != out_features or weight.is_contiguous()
):
replace_parameter(layer, "weight", weight.data.t())
# else: already in [in, out] layout — no-op
weight_scale = layer.weight_scale.t().contiguous()
replace_parameter(layer, "weight_scale", weight_scale.data)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
weight = layer.weight
weight_scale = layer.weight_scale
return torch.ops._xpu_C.fp8_gemm_w8a16(x, weight, weight_scale, bias)
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
pass
class XPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "XPUFp8BlockScaledMM only support on XPU"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module):
super().process_weights_after_loading(layer)
scale_attr = (
"weight_scale_inv" if hasattr(layer, "weight_scale_inv") else "weight_scale"
)
scale = getattr(layer, scale_attr)
# Ragged N (N % block_n != 0): oneDNN needs n_blocks to divide N.
# Weight untouched; only repeat scale rows to a finer N-group gn that
# divides both N and block_n (gn = gcd(N, block_n)):
# scale [ceil(N/block_n), K/block_k] --> [N/gn, K/block_k]
# oneDNN only accepts gn that is a multiple of 16, and gcd(N, block_n)
# is a power of two (block_n=128), so gn must be >= 16. No-op when
# N % block_n == 0.
block_n, block_k = self.weight_group_shape
N, K = layer.weight.shape
if N % block_n != 0:
gn = math.gcd(N, block_n)
assert gn % 16 == 0, (
f"XPU block-scaled FP8: N ({N}) yields group width {gn}, but "
f"oneDNN only supports multiples of 16; this weight shape is "
f"unsupported."
)
col_start = torch.arange(N // gn, device=scale.device) * gn
src_idx = torch.div(col_start, block_n, rounding_mode="floor")
scale = scale.index_select(0, src_idx).contiguous()
# Ragged K needs the runtime activation scale expanded too, which we
# don't handle; DeepSeek/GLM keep K block-aligned, so fail loudly.
assert K % block_k == 0, (
f"XPU block-scaled FP8 requires K ({K}) to be a multiple of the "
f"weight block size ({block_k}); ragged-K weights are unsupported."
)
# Checkpoint scale is [n_blocks, k_blocks] (one value per block tile).
# oneDNN fp8_gemm requires contiguous [k_blocks, n_blocks] layout.
# We store the transposed contiguous buffer as a .t() view so that:
# - MLA's scaled_dequantize still sees [n_blocks, k_blocks] shape
# - apply_block_scaled_mm recovers the contiguous buffer via .t()
scale_kn = scale.data.t().contiguous() # [k_blocks, n_blocks]
replace_parameter(layer, scale_attr, scale_kn.t()) # view: [n_blocks, k_blocks]
if getattr(layer, "is_bmm", False):
self._prepare_bmm_params(layer, scale_kn)
def _prepare_bmm_params(
self, layer: torch.nn.Module, scale_kn: torch.Tensor
) -> None:
"""Precompute batched weight and scale for grouped fp8_bmm (e.g. wo_a).
Splits scale [k_blocks, n_blocks] into [G, k_blocks, n_blocks_per_group]
and weight [N_total, K] into [G, K, N_per_group] for batch GEMM.
"""
batch = layer.bmm_batch_size
k_blocks, n_blocks = scale_kn.shape
layer.bmm_scale = (
scale_kn.reshape(k_blocks, batch, n_blocks // batch)
.permute(1, 0, 2)
.contiguous()
)
w = layer.weight
N_total, K = w.shape
layer.bmm_weight = w.reshape(batch, N_total // batch, K).permute(
0, 2, 1
) # [G, K, N_per_group]
def apply_block_scaled_mm(
self,
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
Bs: torch.Tensor,
) -> torch.Tensor:
# B is [N, K]; .t() gives [K, N] view (no copy).
# Bs is stored as [n_blocks, k_blocks] view; .t() recovers the
# contiguous [k_blocks, n_blocks] buffer that oneDNN expects.
return torch.ops._xpu_C.fp8_gemm(
A,
B.t(),
self.config.out_dtype,
As,
Bs.t(),
torch.Tensor(),
)
@@ -1,36 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .aiter import *
from .cpu import *
from .tilelang import *
from .torch import *
from .triton import *
__all__ = [
"mhc_pre_cuda",
"mhc_post_cuda",
"mhc_fused_post_pre_cuda",
"hc_head_fused_kernel_cuda",
"mhc_pre_aiter",
"mhc_pre_delayed_aiter",
"mhc_fused_post_pre_delayed_rms_norm_aiter",
"mhc_post_aiter",
"mhc_fused_post_pre_aiter",
"hc_head_fused_aiter",
"mhc_pre_cpu",
"mhc_post_cpu",
"hc_head_fused_cpu",
"mhc_pre_tilelang",
"mhc_post_tilelang",
"mhc_fused_post_pre_tilelang",
"hc_head_fused_tilelang",
"mhc_pre_torch",
"mhc_post_torch",
"mhc_fused_post_pre_torch",
"hc_head_fused_torch",
"mhc_pre_mix_triton",
"mhc_pre_triton",
"mhc_post_triton",
"mhc_fused_post_pre_triton",
"hc_head_fused_triton",
]
-428
View File
@@ -1,428 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.utils.torch_utils import direct_register_custom_op
def mhc_pre_aiter(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
n_splits: split-k factor;
norm_weight: optional RMSNorm weight fused into the pre kernel
norm_eps: epsilon for the fused RMSNorm when norm_weight is set
Returns:
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_pre(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_weight,
norm_eps,
)
def _mhc_pre_aiter_fake(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
# Create empty tensors with correct shapes for meta device / shape inference
post_mix = torch.empty(
*outer_shape,
hc_mult,
1,
dtype=torch.float32,
device=residual.device,
)
comb_mix = torch.empty(
*outer_shape,
hc_mult,
hc_mult,
dtype=torch.float32,
device=residual.device,
)
layer_input = torch.empty(
*outer_shape,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
)
return post_mix, comb_mix, layer_input
def mhc_pre_delayed_aiter(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
residual_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""MHC pre with the pre-mix carried in from the previous sublayer.
Matches ``mhc_pre_delayed_torch``: the stream collapse uses *pre_mix*
rather than the gate computed here, and that gate is returned as the
pre-mix for the next sublayer seam.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
pre_mix: shape (..., hc_mult) from the previous sublayer, or None at
model entry to select residual stream zero.
sublayer_out: attention or FFN output, shape (..., hidden_size). When
given with the two mixes below, the preceding post block is
applied here so AITER can fold it into the pre projection.
post_layer_mix: shape (..., hc_mult, 1), post gate for that block.
comb_res_mix: shape (..., hc_mult, hc_mult), residual comb for it.
residual_out: shape (..., hc_mult, hidden_size), written with the post
block's new residual. Required exactly when the post is folded in.
Returns:
post_mix: shape (..., hc_mult, 1), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
next_pre_mix: shape (..., hc_mult), dtype torch.float32
"""
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_pre_delayed(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
sublayer_out,
post_layer_mix,
comb_res_mix,
residual_out,
)
def _mhc_pre_delayed_aiter_fake(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
residual_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
return (
torch.empty(
*outer_shape, hc_mult, 1, dtype=torch.float32, device=residual.device
),
torch.empty(
*outer_shape, hc_mult, hc_mult, dtype=torch.float32, device=residual.device
),
torch.empty(
*outer_shape, hidden_size, dtype=torch.bfloat16, device=residual.device
),
torch.empty(*outer_shape, hc_mult, dtype=torch.float32, device=residual.device),
)
def mhc_fused_post_pre_delayed_rms_norm_aiter(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None,
sublayer_out: torch.Tensor | None,
post_layer_mix: torch.Tensor | None,
comb_res_mix: torch.Tensor | None,
norm_weight: torch.Tensor,
norm_eps: float,
residual_out: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
""":func:`mhc_pre_delayed_aiter` with the RMSNorm of the collapse folded in:
``layer_input`` comes back normalised with ``norm_weight`` / ``norm_eps``."""
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_fused_post_pre_delayed_rms_norm(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
sublayer_out,
post_layer_mix,
comb_res_mix,
norm_weight,
norm_eps,
residual_out,
)
def _mhc_fused_post_pre_delayed_rms_norm_aiter_fake(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None,
sublayer_out: torch.Tensor | None,
post_layer_mix: torch.Tensor | None,
comb_res_mix: torch.Tensor | None,
norm_weight: torch.Tensor,
norm_eps: float,
residual_out: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
return _mhc_pre_delayed_aiter_fake(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
def mhc_post_aiter(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_post(
x,
residual,
post_layer_mix,
comb_res_mix,
)
def _mhc_post_aiter_fake(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
return torch.empty_like(residual)
def mhc_fused_post_pre_aiter(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
tile_n: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Fused mHC post + next mHC pre on ROCm via AITER.
Returns residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur.
"""
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_fused_post_pre(
x,
residual,
post_layer_mix,
comb_res_mix,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
norm_weight,
norm_eps,
)
def _mhc_fused_post_pre_aiter_fake(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
tile_n: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
post_mix = torch.empty(
*outer_shape,
hc_mult,
1,
dtype=torch.float32,
device=residual.device,
)
comb_mix = torch.empty(
*outer_shape,
hc_mult,
hc_mult,
dtype=torch.float32,
device=residual.device,
)
layer_input = torch.empty(
*outer_shape,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
)
return torch.empty_like(residual), post_mix, comb_mix, layer_input
direct_register_custom_op(
op_name="mhc_pre_aiter",
op_func=mhc_pre_aiter,
mutates_args=[],
fake_impl=_mhc_pre_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_pre_delayed_aiter",
op_func=mhc_pre_delayed_aiter,
mutates_args=["residual_out"],
fake_impl=_mhc_pre_delayed_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_fused_post_pre_delayed_rms_norm_aiter",
op_func=mhc_fused_post_pre_delayed_rms_norm_aiter,
mutates_args=["residual_out"],
fake_impl=_mhc_fused_post_pre_delayed_rms_norm_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_post_aiter",
op_func=mhc_post_aiter,
mutates_args=[],
fake_impl=_mhc_post_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_fused_post_pre_aiter",
op_func=mhc_fused_post_pre_aiter,
mutates_args=[],
fake_impl=_mhc_fused_post_pre_aiter_fake,
)
-96
View File
@@ -1,96 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm import _custom_ops as ops
def mhc_pre_cpu(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""CPU-ported mHC pre block (see `mhc_pre_torch` for the eager reference).
The ported kernel (`hc_pre_fused_cpu`) only exposes one merged `hc_eps`
(used for both `hc_pre_eps`/`hc_sinkhorn_eps`) and hardcodes the post-mix
multiplier to 2.0 -- true of every real call site in
`models/deepseek_v4/cpu/model.py`, so this is not a capability loss here.
"""
assert n_splits == 1, "mhc_pre_cpu does not support n_splits != 1"
assert hc_pre_eps == hc_sinkhorn_eps, (
"mhc_pre_cpu requires hc_pre_eps == hc_sinkhorn_eps (single merged hc_eps)"
)
assert hc_post_mult_value == 2.0, "mhc_pre_cpu hardcodes the post-mix multiplier"
hc_mult, hidden_size = residual.shape[-2:]
outer_shape = residual.shape[:-2]
x_flat = residual.reshape(-1, hc_mult, hidden_size)
layer_input, post, comb = ops.hc_pre_fused_cpu(
x_flat,
fn,
hc_scale,
hc_base,
hc_mult,
sinkhorn_repeat,
rms_eps,
hc_pre_eps,
)
post_mix = post.view(*outer_shape, hc_mult, 1)
comb_mix = comb.view(*outer_shape, hc_mult, hc_mult)
layer_input = layer_input.view(*outer_shape, hidden_size)
return post_mix, comb_mix, layer_input
def mhc_post_cpu(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
"""CPU-ported mHC post block (see `mhc_post_torch` for the eager reference)."""
hc_mult, hidden_size = residual.shape[-2:]
outer_shape = residual.shape[:-2]
x_flat = x.reshape(-1, hidden_size)
residual_flat = residual.reshape(-1, hc_mult, hidden_size)
post_flat = post_layer_mix.reshape(-1, hc_mult).float()
comb_flat = comb_res_mix.reshape(-1, hc_mult, hc_mult).float()
out = ops.hc_post_fused_cpu(x_flat, residual_flat, post_flat, comb_flat)
return out.view(*outer_shape, hc_mult, hidden_size)
def hc_head_fused_cpu(
hidden_states: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_norm_eps: float,
hc_eps: float,
) -> torch.Tensor:
"""CPU-ported HC head reduction (see `test_hc_head_cpu` in
tests/kernels/mhc/test_mhc_kernels.py for the eager reference this is tested
against).
The ported kernel's C++ signature takes `(hc_eps, norm_eps)` -- the
opposite order from this wrapper's `(rms_norm_eps, hc_eps)`, which matches
the real call site in `models/deepseek_v4/cpu/model.py`.
"""
hc_mult, hidden_size = hidden_states.shape[-2:]
outer_shape = hidden_states.shape[:-2]
hs_flat = hidden_states.reshape(-1, hc_mult, hidden_size)
out = ops.hc_head_fused_cpu(hs_flat, hc_fn, hc_scale, hc_base, hc_eps, rms_norm_eps)
return out.view(*outer_shape, hidden_size)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-147
View File
@@ -1,147 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
def mhc_pre_delayed_torch(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
x: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Reference for mHC pre using coefficients from the previous sublayer."""
hc_mult = residual.shape[1]
x = (residual.flatten(1) if x is None else x).float()
mixes = (x @ fn.t()) * torch.rsqrt(x.square().mean(-1, keepdim=True) + rms_eps)
pre = (
torch.sigmoid(mixes[:, :hc_mult] * hc_scale[0] + hc_base[:hc_mult]) + hc_pre_eps
)
post = (
torch.sigmoid(
mixes[:, hc_mult : 2 * hc_mult] * hc_scale[1]
+ hc_base[hc_mult : 2 * hc_mult]
)
* hc_post_mult_value
)
comb = mixes[:, 2 * hc_mult :].view(-1, hc_mult, hc_mult) * hc_scale[2]
comb = comb + hc_base[2 * hc_mult :].view(1, hc_mult, hc_mult)
comb = torch.softmax(comb, dim=-1) + hc_sinkhorn_eps
comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
for _ in range(sinkhorn_repeat - 1):
comb = comb / (comb.sum(dim=-1, keepdim=True) + hc_sinkhorn_eps)
comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
layer_input = (
residual[:, 0]
if pre_mix is None
else (pre_mix.unsqueeze(-1) * residual.float()).sum(dim=1).to(residual.dtype)
)
return post.unsqueeze(-1), comb, layer_input, pre
def mhc_pre_torch(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Forward pass for mHC pre block.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
n_splits: split-k factor;
Returns:
post_mix: shape (..., hc_mult), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
"""
# Validate shapes
assert residual.dtype == torch.bfloat16
assert fn.dtype == torch.float32
assert hc_scale.dtype == torch.float32
assert hc_base.dtype == torch.float32
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
hc_mult2 = hc_mult * hc_mult
hc_mult3 = hc_mult * 2 + hc_mult2
hc_hidden_size = hc_mult * hidden_size
assert fn.shape[0] == hc_mult3
assert fn.shape[1] == hc_hidden_size
assert hc_scale.shape == (3,)
assert hc_base.shape == (hc_mult3,)
outer_shape = residual.shape[:-2]
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
fn_flat = fn
x = residual_flat.view(num_tokens, hc_mult * hidden_size).to(torch.float32)
mixes = torch.matmul(x, fn_flat.t())
sqrsum = x.square().sum(dim=-1, keepdim=True)
mixes = mixes * torch.rsqrt(sqrsum / (hc_mult * hidden_size) + rms_eps)
pre_logits = mixes[:, :hc_mult] * hc_scale[0] + hc_base[:hc_mult]
pre_mix = torch.sigmoid(pre_logits) + hc_pre_eps
post_logits = (
mixes[:, hc_mult : 2 * hc_mult] * hc_scale[1] + hc_base[hc_mult : 2 * hc_mult]
)
post_mix = torch.sigmoid(post_logits) * hc_post_mult_value
comb_logits = mixes[:, 2 * hc_mult :].view(num_tokens, hc_mult, hc_mult) * hc_scale[
2
] + hc_base[2 * hc_mult :].view(1, hc_mult, hc_mult)
comb_mix = torch.softmax(comb_logits, dim=-1) + hc_sinkhorn_eps
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
for _ in range(sinkhorn_repeat - 1):
comb_mix = comb_mix / (comb_mix.sum(dim=-1, keepdim=True) + hc_sinkhorn_eps)
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
layer_input = torch.sum(
pre_mix.unsqueeze(-1) * residual_flat.to(torch.float32), dim=1
).to(torch.bfloat16)
return (
post_mix.view(*outer_shape, hc_mult, 1),
comb_mix.view(*outer_shape, hc_mult, hc_mult),
layer_input.view(*outer_shape, hidden_size),
)
def mhc_post_torch(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
mixed_residual = torch.einsum(
"...ij,...ih->...jh",
comb_res_mix.to(torch.float32),
residual.to(torch.float32),
)
post_term = post_layer_mix.to(torch.float32) * x.unsqueeze(-2).to(torch.float32)
return (mixed_residual + post_term).to(residual.dtype)
-349
View File
@@ -1,349 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn.functional as F
from torch import Tensor
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
@triton.jit
def _rmsnorm_nw_kernel(
x_ptr,
out_ptr,
stride_row,
D,
eps,
RBLOCK: tl.constexpr,
):
"""Weight-free RMSNorm Triton kernel: out = x * rsqrt(mean(x², -1) + eps)."""
row = tl.program_id(0)
cols = tl.arange(0, RBLOCK)
mask = cols < D
x = tl.load(
x_ptr + row * stride_row + cols,
mask=mask,
other=0.0,
eviction_policy="evict_first",
).to(tl.float32)
var = tl.sum(x * x, 0) / D
rstd = tl.rsqrt(var + eps)
out = (x * rstd).to(out_ptr.dtype.element_ty)
tl.store(out_ptr + row * D + cols, out, mask=mask, eviction_policy="evict_first")
def rmsnorm_nw(x: Tensor, eps: float) -> Tensor:
"""Weight-free RMSNorm over the last dimension.
Treats *x* as ``[num_rows, D]`` where ``num_rows = product(shape[:-1])``.
Returns a contiguous tensor with the same shape and dtype as *x*.
"""
orig_shape = x.shape
D = orig_shape[-1]
x_2d = x.reshape(-1, D)
num_rows = x_2d.shape[0]
out = torch.empty_like(x_2d)
RBLOCK = triton.next_power_of_2(D)
_rmsnorm_nw_kernel[(num_rows,)](
x_2d,
out,
x_2d.stride(0),
D,
eps,
RBLOCK=RBLOCK,
num_warps=1 if RBLOCK <= 512 else (4 if RBLOCK <= 4096 else 8),
)
return out.view(orig_shape)
@triton.jit
def _hc_head_reduce_store_kernel(
pre_ptr,
x_ptr,
out_ptr,
hidden_size: tl.constexpr,
hc_mult: tl.constexpr,
pre_stride_t: tl.constexpr,
pre_stride_m: tl.constexpr,
x_stride_t: tl.constexpr,
x_stride_m: tl.constexpr,
x_stride_h: tl.constexpr,
out_stride_t: tl.constexpr,
out_stride_h: tl.constexpr,
BLOCK_H: tl.constexpr,
):
token_idx = tl.program_id(0).to(tl.int64)
block_idx = tl.program_id(1)
offsets = block_idx * BLOCK_H + tl.arange(0, BLOCK_H)
mask = offsets < hidden_size
acc = tl.zeros((BLOCK_H,), dtype=tl.float32)
for mix_idx in tl.static_range(0, hc_mult):
pre = tl.load(pre_ptr + token_idx * pre_stride_t + mix_idx * pre_stride_m).to(
tl.float32
)
x = tl.load(
x_ptr
+ token_idx * x_stride_t
+ mix_idx * x_stride_m
+ offsets * x_stride_h,
mask=mask,
other=0.0,
).to(tl.float32)
acc += pre * x
tl.store(
out_ptr + token_idx * out_stride_t + offsets * out_stride_h,
acc,
mask=mask,
)
def hc_collapse_triton(x: Tensor, pre_mix: Tensor) -> Tensor:
"""Collapse BF16 residual streams with FP32 pre-mix coefficients."""
assert x.ndim == 3 and x.dtype == torch.bfloat16
num_tokens, hc_mult, hidden_size = x.shape
assert pre_mix.shape == (num_tokens, hc_mult)
assert pre_mix.dtype == torch.float32
out = torch.empty(num_tokens, hidden_size, dtype=x.dtype, device=x.device)
if num_tokens == 0:
return out
block_h = 1024
_hc_head_reduce_store_kernel[(num_tokens, triton.cdiv(hidden_size, block_h))](
pre_mix,
x,
out,
hidden_size,
hc_mult,
pre_mix.stride(0),
pre_mix.stride(1),
x.stride(0),
x.stride(1),
x.stride(2),
out.stride(0),
out.stride(1),
BLOCK_H=block_h,
num_warps=4,
# Preserve the separate FP32 multiply and sum in the Torch reference.
enable_fp_fusion=False,
)
return out
def _hc_collapse_triton_fake(x: Tensor, pre_mix: Tensor) -> Tensor:
return torch.empty(x.shape[0], x.shape[2], dtype=x.dtype, device=x.device)
@triton.jit
def _mhc_pre_mix_kernel(
gemm_ptr,
sqrsum_ptr,
hc_scale_ptr,
hc_base_ptr,
out_ptr,
splitk,
hc_mult: tl.constexpr,
gemm_stride_k,
gemm_stride_t,
gemm_stride_j,
sqrsum_stride_k,
sqrsum_stride_t,
out_stride_t,
out_stride_j,
inv_hc_hidden,
rms_eps,
hc_pre_eps,
SPLITK_BLOCK: tl.constexpr,
HC_BLOCK: tl.constexpr,
):
"""Recover the pre-mix gate from a split-k mHC pre GEMM output."""
token_idx = tl.program_id(0).to(tl.int64)
ks = tl.arange(0, SPLITK_BLOCK)
js = tl.arange(0, HC_BLOCK)
kmask = ks < splitk
jmask = js < hc_mult
# Only the first hc_mult GEMM columns feed the pre gate.
gemm = tl.load(
gemm_ptr
+ ks[:, None] * gemm_stride_k
+ token_idx * gemm_stride_t
+ js[None, :] * gemm_stride_j,
mask=kmask[:, None] & jmask[None, :],
other=0.0,
).to(tl.float32)
mixes = tl.sum(gemm, 0)
sqrsum = tl.load(
sqrsum_ptr + ks * sqrsum_stride_k + token_idx * sqrsum_stride_t,
mask=kmask,
other=0.0,
).to(tl.float32)
rstd = tl.rsqrt(tl.sum(sqrsum, 0) * inv_hc_hidden + rms_eps)
scale = tl.load(hc_scale_ptr).to(tl.float32)
base = tl.load(hc_base_ptr + js, mask=jmask, other=0.0).to(tl.float32)
pre = tl.sigmoid(mixes * rstd * scale + base) + hc_pre_eps
tl.store(out_ptr + token_idx * out_stride_t + js * out_stride_j, pre, mask=jmask)
def mhc_pre_mix_triton(
gemm_out: Tensor,
sqrsum: Tensor,
hc_scale: Tensor,
hc_base: Tensor,
hc_mult: int,
hc_hidden_size: int,
rms_eps: float,
hc_pre_eps: float,
) -> Tensor:
"""Pre-mix gate for the delayed mHC pre, from AITER's split-k GEMM output.
AITER's ``mhc_pre_big_fuse`` consumes the unreduced ``[splitk, tokens,
hc_mult3]`` GEMM output and the matching row square-sums, but only returns
the post and comb gates. The delayed formulation also needs the pre gate,
to carry into the next sublayer seam. It is the same slice of the same
numbers, so recover it here rather than repeating the projection.
"""
assert gemm_out.ndim == 3 and gemm_out.dtype == torch.float32
assert sqrsum.ndim == 2 and sqrsum.dtype == torch.float32
splitk, num_tokens = gemm_out.shape[0], gemm_out.shape[1]
assert sqrsum.shape == (splitk, num_tokens)
out = torch.empty(num_tokens, hc_mult, dtype=torch.float32, device=gemm_out.device)
if num_tokens == 0:
return out
_mhc_pre_mix_kernel[(num_tokens,)](
gemm_out,
sqrsum,
hc_scale,
hc_base,
out,
splitk,
hc_mult,
gemm_out.stride(0),
gemm_out.stride(1),
gemm_out.stride(2),
sqrsum.stride(0),
sqrsum.stride(1),
out.stride(0),
out.stride(1),
1.0 / hc_hidden_size,
rms_eps,
hc_pre_eps,
SPLITK_BLOCK=triton.next_power_of_2(splitk),
HC_BLOCK=triton.next_power_of_2(hc_mult),
num_warps=1,
# Match the separate FP32 multiply and add in the Torch reference.
enable_fp_fusion=False,
)
return out
def _mhc_pre_mix_triton_fake(
gemm_out: Tensor,
sqrsum: Tensor,
hc_scale: Tensor,
hc_base: Tensor,
hc_mult: int,
hc_hidden_size: int,
rms_eps: float,
hc_pre_eps: float,
) -> Tensor:
return torch.empty(
gemm_out.shape[1], hc_mult, dtype=torch.float32, device=gemm_out.device
)
def hc_head_reduce_triton_kernel(
x: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
out: torch.Tensor,
norm_eps: float,
hc_eps: float,
) -> None:
x_flat = x.flatten(-2)
x_normed = rmsnorm_nw(x_flat, norm_eps)
mixes = F.linear(x_normed.float(), hc_fn)
pre = torch.sigmoid(mixes * hc_scale + hc_base) + hc_eps
hidden_size = x.shape[-1]
hc_mult = x.shape[-2]
block_h = 1024
_hc_head_reduce_store_kernel[(x.shape[0], (hidden_size + block_h - 1) // block_h)](
pre,
x,
out,
hidden_size,
hc_mult,
pre.stride(0),
pre.stride(1),
x.stride(0),
x.stride(1),
x.stride(2),
out.stride(0),
out.stride(1),
BLOCK_H=block_h,
num_warps=4,
)
def _hc_head_triton(
hs_flat: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
out: torch.Tensor,
hidden_size: int,
rms_eps: float,
hc_eps: float,
hc_mult: int,
) -> None:
"""Fill pre-allocated `out` (T, H) in-place with the hc_head result."""
if hs_flat.shape[0] == 0:
return
hc_head_reduce_triton_kernel(
hs_flat,
fn,
hc_scale,
hc_base,
out,
rms_eps,
hc_eps,
)
return
direct_register_custom_op(
op_name="hc_head_triton",
op_func=_hc_head_triton,
mutates_args=["out"],
)
direct_register_custom_op(
op_name="hc_collapse_triton",
op_func=hc_collapse_triton,
mutates_args=[],
fake_impl=_hc_collapse_triton_fake,
)
direct_register_custom_op(
op_name="mhc_pre_mix_triton",
op_func=mhc_pre_mix_triton,
mutates_args=[],
fake_impl=_mhc_pre_mix_triton_fake,
)
-101
View File
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from dataclasses import asdict, dataclass
from typing import Any
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
compute_num_split,
mhc_pre_big_fuse_with_norm_tilelang,
)
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.math_utils import cdiv
def compute_mhc_pre_num_splits(input_size: int, num_tokens: int) -> int:
splits = compute_num_split(64, input_size, cdiv(num_tokens, 64))
# Bound both GEMM and fused-normalization specializations during startup.
return 1 if splits == 1 else 4 if splits <= 4 else 16
class MHCPreNormKernel(VllmJitKernel["MHCPreNormKernel.CompileKey"]):
@dataclass(frozen=True)
class CompileKey:
hidden_size: int
rms_eps: float
hc_pre_eps: float
hc_sinkhorn_eps: float
hc_post_mult_value: float
sinkhorn_repeat: int
norm_eps: float
n_splits: int
hc_mult: int
use_pre_mix_in: bool
rms_numel: int
save_pre_mix: bool = True
split_mode: str = "fused"
write_aux: bool = False
kernel: Any = staticmethod(mhc_pre_big_fuse_with_norm_tilelang)
def dispatch(self, *, n_splits, **fields) -> CompileKey: # type: ignore[override]
return self.CompileKey(n_splits=n_splits, **fields)
def get_warmup_keys(
self,
*,
max_tokens: int,
extra_splits: Iterable[int] = (),
**fields,
) -> list[CompileKey]:
# The split heuristic changes only at 64-token boundaries. Callers that
# reach this kernel through another GEMM (the fused post + pre-norm
# GEMM) pass the splits that GEMM picks in extra_splits.
splits = sorted(
set(extra_splits)
| (
{
compute_mhc_pre_num_splits(fields["rms_numel"], num_tokens)
for num_tokens in range(1, max_tokens + 1, 64)
}
if is_deep_gemm_supported()
else {1}
)
)
return self._trace_dispatch(self.dispatch)(n_splits=splits, **fields)
def compile(self, compile_key: CompileKey) -> None:
if compile_key not in self._compiled_cache:
self._compiled_cache[compile_key] = self.kernel.compile(
**asdict(compile_key)
)
def compile_many(self, compile_keys: Iterable[CompileKey]) -> None:
missing = list(
dict.fromkeys(
key for key in compile_keys if key not in self._compiled_cache
)
)
if len(missing) < 2:
return super().compile_many(missing)
# TileLang elaborates serially before compiling independent TIR modules.
compiled = self.kernel.par_compile(
[asdict(key) for key in missing], num_workers=min(4, len(missing))
)
self._compiled_cache.update(zip(missing, compiled, strict=True))
def __call__(self, *tensors, **fields):
compile_key = self.dispatch(
n_splits=tensors[0].shape[0],
**fields,
)
if not current_platform.is_cuda():
return self.kernel(*tensors, **asdict(compile_key))
return self._get_or_compile(compile_key)(*tensors)
MHC_PRE_NORM_KERNEL = MHCPreNormKernel()