[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,602 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import Float32, Int32, Uint32, Uint64
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import recast_val
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
|
||||
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
|
||||
CuTeDSLLaunchSpec,
|
||||
VllmCuTeDSLJitKernel,
|
||||
)
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
def _dcp_merge_topk_and_interleave(vllm_config: Any) -> tuple[int, int]:
|
||||
"""(topk, cp_interleave) the DCP merge runs with.
|
||||
|
||||
Kpool indexers (GLM-5.x) merge pool ids: topk and the interleave are both
|
||||
in units of ``index_kpool`` tokens.
|
||||
"""
|
||||
hf_text_config = vllm_config.model_config.hf_text_config
|
||||
topk = hf_text_config.index_topk
|
||||
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
|
||||
kpool = getattr(hf_text_config, "index_kpool", None) or 1
|
||||
if kpool > 1:
|
||||
topk //= kpool
|
||||
cp_interleave //= kpool
|
||||
return topk, cp_interleave
|
||||
|
||||
|
||||
def stable_topk_from_gathered_candidates_cutedsl(
|
||||
gathered: torch.Tensor,
|
||||
topk: int,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if out is None:
|
||||
out = torch.empty(
|
||||
(gathered.shape[0], topk),
|
||||
dtype=torch.int32,
|
||||
device=gathered.device,
|
||||
)
|
||||
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL(gathered, out, topk=topk)
|
||||
return out
|
||||
|
||||
|
||||
def pack_dcp_topk_candidates_cutedsl(
|
||||
logits: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
packed: torch.Tensor,
|
||||
dcp_rank: int,
|
||||
dcp_world_size: int,
|
||||
cp_interleave: int,
|
||||
row_starts: torch.Tensor | None,
|
||||
) -> None:
|
||||
topk = topk_indices.shape[1]
|
||||
row_starts_arg = row_starts if row_starts is not None else topk_indices
|
||||
_PACK_DCP_TOPK_CANDIDATES_KERNEL(
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
row_starts_arg,
|
||||
logits_stride0=logits.stride(0),
|
||||
logits_stride1=logits.stride(1),
|
||||
topk_stride0=topk_indices.stride(0),
|
||||
topk_stride1=topk_indices.stride(1),
|
||||
packed_stride0=packed.stride(0),
|
||||
packed_stride1=packed.stride(1),
|
||||
packed_stride2=packed.stride(2),
|
||||
num_cols=logits.shape[1],
|
||||
dcp_rank=dcp_rank,
|
||||
dcp_world_size=dcp_world_size,
|
||||
cp_interleave=cp_interleave,
|
||||
has_row_starts=row_starts is not None,
|
||||
topk=topk,
|
||||
block_size=512,
|
||||
)
|
||||
|
||||
|
||||
class PackDCPTopkCandidatesKernel(
|
||||
VllmTritonJitKernel["PackDCPTopkCandidatesKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dcp_rank: int
|
||||
dcp_world_size: int
|
||||
cp_interleave: int
|
||||
has_row_starts: bool
|
||||
topk: int
|
||||
block_size: int
|
||||
|
||||
# These scalars only describe runtime layouts and bounds; the constexpr
|
||||
# fields below own the launch geometry and algorithmic specialization.
|
||||
@staticmethod
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"logits_stride0",
|
||||
"logits_stride1",
|
||||
"topk_stride0",
|
||||
"topk_stride1",
|
||||
"packed_stride0",
|
||||
"packed_stride1",
|
||||
"packed_stride2",
|
||||
"num_cols",
|
||||
]
|
||||
)
|
||||
def kernel(
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
row_starts,
|
||||
logits_stride0,
|
||||
logits_stride1,
|
||||
topk_stride0,
|
||||
topk_stride1,
|
||||
packed_stride0,
|
||||
packed_stride1,
|
||||
packed_stride2,
|
||||
num_cols,
|
||||
DCP_RANK: tl.constexpr,
|
||||
DCP_WORLD_SIZE: tl.constexpr,
|
||||
CP_INTERLEAVE: tl.constexpr,
|
||||
HAS_ROW_STARTS: tl.constexpr,
|
||||
TOPK: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
tile = tl.program_id(1)
|
||||
cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = cols < TOPK
|
||||
|
||||
local_idx = tl.load(
|
||||
topk_indices + row * topk_stride0 + cols * topk_stride1,
|
||||
mask=mask,
|
||||
other=-1,
|
||||
)
|
||||
valid = local_idx >= 0
|
||||
safe_local_idx = tl.maximum(local_idx, 0)
|
||||
|
||||
row_start = 0
|
||||
if HAS_ROW_STARTS:
|
||||
row_start = tl.load(row_starts + row)
|
||||
|
||||
score_col = safe_local_idx + row_start
|
||||
score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0))
|
||||
score = tl.load(
|
||||
logits + row * logits_stride0 + score_col * logits_stride1,
|
||||
mask=mask & valid,
|
||||
other=-float("inf"),
|
||||
)
|
||||
|
||||
global_id = (
|
||||
(safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE)
|
||||
+ DCP_RANK * CP_INTERLEAVE
|
||||
+ safe_local_idx % CP_INTERLEAVE
|
||||
)
|
||||
global_id = tl.where(valid, global_id, -1)
|
||||
|
||||
packed_base = packed + row * packed_stride0 + cols * packed_stride1
|
||||
tl.store(packed_base, score, mask=mask)
|
||||
tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
has_row_starts: bool,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
has_row_starts=has_row_starts,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if dcp_world_size <= 1:
|
||||
return []
|
||||
topk, cp_interleave = _dcp_merge_topk_and_interleave(vllm_config)
|
||||
if topk <= 0:
|
||||
return []
|
||||
|
||||
try:
|
||||
from vllm.distributed.parallel_state import get_dcp_group
|
||||
|
||||
dcp_rank = get_dcp_group().rank_in_group
|
||||
except Exception:
|
||||
dcp_rank = 0
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dcp_rank=dcp_rank,
|
||||
dcp_world_size=dcp_world_size,
|
||||
cp_interleave=cp_interleave,
|
||||
has_row_starts=(False, True),
|
||||
topk=topk,
|
||||
block_size=512,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
fp32_ptr = TritonWarmupTensor(torch.float32)
|
||||
int32_ptr = TritonWarmupTensor(torch.int32)
|
||||
return dict(
|
||||
logits=fp32_ptr,
|
||||
topk_indices=TritonWarmupTensor(torch.int32, shape=(1, compile_key.topk)),
|
||||
packed=fp32_ptr,
|
||||
row_starts_arg=int32_ptr,
|
||||
logits_stride0=1,
|
||||
logits_stride1=1,
|
||||
topk_stride0=1,
|
||||
topk_stride1=1,
|
||||
packed_stride0=1,
|
||||
packed_stride1=1,
|
||||
packed_stride2=1,
|
||||
num_cols=1,
|
||||
dcp_rank=compile_key.dcp_rank,
|
||||
dcp_world_size=compile_key.dcp_world_size,
|
||||
cp_interleave=compile_key.cp_interleave,
|
||||
has_row_starts=compile_key.has_row_starts,
|
||||
topk=compile_key.topk,
|
||||
block_size=compile_key.block_size,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
packed: torch.Tensor,
|
||||
row_starts_arg: torch.Tensor,
|
||||
*,
|
||||
logits_stride0: int,
|
||||
logits_stride1: int,
|
||||
topk_stride0: int,
|
||||
topk_stride1: int,
|
||||
packed_stride0: int,
|
||||
packed_stride1: int,
|
||||
packed_stride2: int,
|
||||
num_cols: int,
|
||||
dcp_rank: int,
|
||||
dcp_world_size: int,
|
||||
cp_interleave: int,
|
||||
has_row_starts: bool,
|
||||
topk: int,
|
||||
block_size: int,
|
||||
) -> LaunchSpec:
|
||||
grid = (topk_indices.shape[0], triton.cdiv(topk, block_size))
|
||||
return grid, dict(
|
||||
row_starts=row_starts_arg,
|
||||
DCP_RANK=dcp_rank,
|
||||
DCP_WORLD_SIZE=dcp_world_size,
|
||||
CP_INTERLEAVE=cp_interleave,
|
||||
HAS_ROW_STARTS=has_row_starts,
|
||||
TOPK=topk,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=8,
|
||||
)
|
||||
|
||||
|
||||
class StableTopKFromGatheredCandidatesKernel(
|
||||
VllmCuTeDSLJitKernel["StableTopKFromGatheredCandidatesKernel.CompileKey"]
|
||||
):
|
||||
tb_size = 512
|
||||
hist_bins = 2048
|
||||
radix_bits = (hist_bins - 1).bit_length()
|
||||
key_bits = Uint64.width
|
||||
radix_passes = (key_bits + radix_bits - 1) // radix_bits
|
||||
final_radix_bits = key_bits - radix_bits * (radix_passes - 1)
|
||||
hist_chunks = (hist_bins + tb_size - 1) // tb_size
|
||||
warps_per_block = tb_size // cute.arch.WARP_SIZE
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
topk: int
|
||||
num_candidates: int
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> Any:
|
||||
tb_size = StableTopKFromGatheredCandidatesKernel.tb_size
|
||||
hist_bins = StableTopKFromGatheredCandidatesKernel.hist_bins
|
||||
radix_bits = StableTopKFromGatheredCandidatesKernel.radix_bits
|
||||
key_bits = StableTopKFromGatheredCandidatesKernel.key_bits
|
||||
radix_passes = StableTopKFromGatheredCandidatesKernel.radix_passes
|
||||
final_radix_bits = StableTopKFromGatheredCandidatesKernel.final_radix_bits
|
||||
hist_chunks = StableTopKFromGatheredCandidatesKernel.hist_chunks
|
||||
warps_per_block = StableTopKFromGatheredCandidatesKernel.warps_per_block
|
||||
topk = compile_key.topk
|
||||
assert hist_bins == 1 << radix_bits
|
||||
assert compile_key.num_candidates % tb_size == 0, (
|
||||
"StableTopKFromGatheredCandidatesKernel requires candidate count "
|
||||
f"to be a multiple of {tb_size}, got {compile_key.num_candidates}"
|
||||
)
|
||||
keys_per_thread = compile_key.num_candidates // tb_size
|
||||
|
||||
@cute.struct
|
||||
class SharedStorage:
|
||||
hist: cute.struct.MemRange[Int32, hist_bins]
|
||||
committed_count: cute.struct.MemRange[Int32, 1]
|
||||
running_count: cute.struct.MemRange[Int32, 1]
|
||||
threshold_bin: cute.struct.MemRange[Int32, 1]
|
||||
threshold_found: cute.struct.MemRange[Int32, 1]
|
||||
include_threshold_bin: cute.struct.MemRange[Int32, 1]
|
||||
prefix_s: cute.struct.Align[cute.struct.MemRange[Uint64, 1], 8]
|
||||
warp_totals: cute.struct.MemRange[Int32, warps_per_block]
|
||||
|
||||
shared_storage = SharedStorage
|
||||
|
||||
@cute.jit
|
||||
def warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32:
|
||||
for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1):
|
||||
offset = 1 << i
|
||||
partial = cute.arch.shuffle_sync_up(
|
||||
val, offset=offset, mask_and_clamp=0
|
||||
)
|
||||
if lane >= offset:
|
||||
val += partial
|
||||
return val
|
||||
|
||||
@cute.jit
|
||||
def block_scan_inclusive_i32(
|
||||
val: Int32,
|
||||
lane: Int32,
|
||||
warp_id: Int32,
|
||||
warp_scratch: cute.Tensor,
|
||||
warps_per_block: int,
|
||||
) -> Int32:
|
||||
prefix = warp_scan_inclusive_i32(val, lane)
|
||||
if lane == Int32(cute.arch.WARP_SIZE - 1):
|
||||
warp_scratch[0, warp_id] = prefix
|
||||
cute.arch.sync_threads()
|
||||
|
||||
if warp_id == Int32(0):
|
||||
warp_total = Int32(0)
|
||||
if lane < Int32(warps_per_block):
|
||||
warp_total = warp_scratch[0, lane]
|
||||
warp_prefix = warp_scan_inclusive_i32(warp_total, lane)
|
||||
if lane < Int32(warps_per_block):
|
||||
warp_scratch[0, lane] = warp_prefix - warp_total
|
||||
cute.arch.sync_threads()
|
||||
|
||||
return prefix + warp_scratch[0, warp_id]
|
||||
|
||||
@cute.jit
|
||||
def stable_key(score: Float32, token_id: Int32) -> Uint64:
|
||||
bits = recast_val(score, Uint32)
|
||||
mask = Uint32(0x80000000)
|
||||
if (bits & Uint32(0x80000000)) != Uint32(0):
|
||||
mask = Uint32(0xFFFFFFFF)
|
||||
score_key = Uint64(bits ^ mask) << Uint64(32)
|
||||
id_key = Uint64(~Uint32(token_id))
|
||||
key = score_key | id_key
|
||||
if token_id < Int32(0):
|
||||
key = Uint64(0)
|
||||
return key
|
||||
|
||||
@cute.jit
|
||||
def prefix_matches(
|
||||
key: Uint64,
|
||||
prefix: Uint64,
|
||||
prefix_bits: Int32,
|
||||
):
|
||||
matches = prefix_bits == Int32(0)
|
||||
if prefix_bits != Int32(0):
|
||||
shift = Int32(key_bits) - prefix_bits
|
||||
matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift))
|
||||
return matches
|
||||
|
||||
@cute.jit
|
||||
def radix_pass(
|
||||
keys: cute.Tensor,
|
||||
output: cute.Tensor,
|
||||
storage,
|
||||
tid: Int32,
|
||||
step: Int32,
|
||||
bits: int,
|
||||
is_final_pass: bool,
|
||||
):
|
||||
hist_smem = storage.hist.get_tensor(cute.make_layout((hist_bins,)))
|
||||
committed_count_smem = storage.committed_count.data_ptr()
|
||||
running_count_smem = storage.running_count.data_ptr()
|
||||
threshold_bin_smem = storage.threshold_bin.data_ptr()
|
||||
threshold_found_smem = storage.threshold_found.data_ptr()
|
||||
include_threshold_bin_smem = storage.include_threshold_bin.data_ptr()
|
||||
prefix_smem = storage.prefix_s.data_ptr()
|
||||
warp_totals_smem = storage.warp_totals.get_tensor(
|
||||
cute.make_layout((1, warps_per_block))
|
||||
)
|
||||
|
||||
prefix_bits = step * Int32(radix_bits)
|
||||
num_bins = 1 << bits
|
||||
block_scan_iterations = (num_bins + tb_size - 1) // tb_size
|
||||
shift = Int32(key_bits) - prefix_bits - Int32(bits)
|
||||
bin_mask = Uint64(num_bins - 1)
|
||||
prefix = prefix_smem.load()
|
||||
|
||||
for chunk in cutlass.range_constexpr(hist_chunks):
|
||||
hist_smem[tid + Int32(chunk * tb_size)] = Int32(0)
|
||||
if tid == Int32(0):
|
||||
running_count_smem.store(committed_count_smem.load())
|
||||
include_threshold_bin_smem.store(Int32(0))
|
||||
threshold_found_smem.store(Int32(0))
|
||||
cute.arch.sync_threads()
|
||||
|
||||
for key_idx in cutlass.range_constexpr(keys_per_thread):
|
||||
key = keys[key_idx]
|
||||
if prefix_matches(key, prefix, prefix_bits):
|
||||
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
|
||||
cute.arch.atomic_add(
|
||||
hist_smem.iterator + bin_idx,
|
||||
Int32(1),
|
||||
sem="relaxed",
|
||||
scope="cta",
|
||||
)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
lane = cute.arch.lane_idx()
|
||||
warp_id = cute.arch.warp_idx()
|
||||
# Each iteration scans one tb_size-wide slice of bins, high to low.
|
||||
iter = Int32(0)
|
||||
threshold_found = threshold_found_smem.load()
|
||||
while threshold_found == Int32(0) and iter < Int32(block_scan_iterations):
|
||||
bin_idx = Int32(num_bins - 1) - (iter * Int32(tb_size) + tid)
|
||||
count = hist_smem[bin_idx]
|
||||
chunk_inclusive = block_scan_inclusive_i32(
|
||||
count,
|
||||
lane,
|
||||
warp_id,
|
||||
warp_totals_smem,
|
||||
warps_per_block,
|
||||
)
|
||||
running_count = running_count_smem.load()
|
||||
prior_in_scan_slice = chunk_inclusive - count
|
||||
remaining = Int32(topk) - running_count - prior_in_scan_slice
|
||||
if count > Int32(0) and remaining > Int32(0) and remaining <= count:
|
||||
threshold_bin_smem.store(bin_idx)
|
||||
if count <= remaining or cutlass.const_expr(is_final_pass):
|
||||
include_threshold_bin_smem.store(Int32(1))
|
||||
threshold_found_smem.store(Int32(1))
|
||||
# Barrier: every thread must finish reading running_count for this
|
||||
# slice before tb_size-1 advances it, else a warp racing ahead to
|
||||
# the store makes a lagging thread double-count the slice total
|
||||
# (-> remaining too small -> threshold too high -> under-fill).
|
||||
cute.arch.sync_threads()
|
||||
if tid == Int32(tb_size - 1):
|
||||
running_count_smem.store(running_count + chunk_inclusive)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
threshold_found = threshold_found_smem.load()
|
||||
iter += Int32(1)
|
||||
|
||||
threshold = threshold_bin_smem.load()
|
||||
should_include_threshold = include_threshold_bin_smem.load() != Int32(0)
|
||||
for key_idx in cutlass.range_constexpr(keys_per_thread):
|
||||
key = keys[key_idx]
|
||||
if prefix_matches(key, prefix, prefix_bits):
|
||||
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
|
||||
selected = bin_idx > threshold
|
||||
if should_include_threshold:
|
||||
selected = selected or bin_idx == threshold
|
||||
if selected:
|
||||
dst = cute.arch.atomic_add(
|
||||
committed_count_smem,
|
||||
Int32(1),
|
||||
sem="relaxed",
|
||||
scope="cta",
|
||||
)
|
||||
if dst < Int32(topk):
|
||||
output[dst] = recast_val(~Uint32(key), Int32)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
pass_finished = include_threshold_bin_smem.load()
|
||||
if tid == Int32(0) and pass_finished == Int32(0):
|
||||
prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift)))
|
||||
cute.arch.sync_threads()
|
||||
return pass_finished
|
||||
|
||||
@cute.kernel
|
||||
def device_kernel(input: cute.Tensor, out: cute.Tensor):
|
||||
row, _, _ = cute.arch.block_idx()
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
input_row = input[row, None, None]
|
||||
output_row = out[row, None]
|
||||
keys = cute.make_rmem_tensor((keys_per_thread,), Uint64)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
storage = smem.allocate(shared_storage, 8)
|
||||
committed_count_smem = storage.committed_count.data_ptr()
|
||||
prefix_smem = storage.prefix_s.data_ptr()
|
||||
for i in range(tid, topk, tb_size):
|
||||
output_row[i] = Int32(-1)
|
||||
|
||||
for key_idx in cutlass.range_constexpr(keys_per_thread):
|
||||
col = tid + Int32(key_idx * tb_size)
|
||||
score = Float32(input_row[col, 0])
|
||||
token_id = Int32(input_row[col, 1])
|
||||
keys[key_idx] = stable_key(score, token_id)
|
||||
|
||||
if tid == Int32(0):
|
||||
committed_count_smem.store(Int32(0))
|
||||
prefix_smem.store(Uint64(0))
|
||||
cute.arch.sync_threads()
|
||||
|
||||
step = Int32(0)
|
||||
finished = Int32(0)
|
||||
while finished == Int32(0) and step < Int32(radix_passes - 1):
|
||||
finished = radix_pass(
|
||||
keys,
|
||||
output_row,
|
||||
storage,
|
||||
tid,
|
||||
step,
|
||||
radix_bits,
|
||||
False,
|
||||
)
|
||||
step += Int32(1)
|
||||
|
||||
if finished == Int32(0):
|
||||
radix_pass(
|
||||
keys,
|
||||
output_row,
|
||||
storage,
|
||||
tid,
|
||||
Int32(radix_passes - 1),
|
||||
final_radix_bits,
|
||||
True,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def host_entrypoint(
|
||||
gathered: cute.Tensor,
|
||||
out: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
grid = (gathered.shape[0], 1, 1)
|
||||
device_kernel(gathered, out).launch(
|
||||
grid=grid,
|
||||
block=(tb_size, 1, 1),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
return host_entrypoint
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(**compile_key_fields)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if dcp_world_size <= 1:
|
||||
return []
|
||||
topk, _ = _dcp_merge_topk_and_interleave(vllm_config)
|
||||
if topk <= 0:
|
||||
return []
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
topk=topk,
|
||||
num_candidates=topk * dcp_world_size,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
|
||||
num_rows = cute.sym_int()
|
||||
gathered = cute.runtime.make_fake_tensor(
|
||||
Float32,
|
||||
(num_rows, compile_key.num_candidates, 2),
|
||||
stride=(cute.sym_int64(divisibility=2), 2, 1),
|
||||
assumed_align=8,
|
||||
)
|
||||
out = make_fake_tensor(
|
||||
Int32,
|
||||
(num_rows, compile_key.topk),
|
||||
divisibility=1,
|
||||
)
|
||||
return gathered, out
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
gathered: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
*,
|
||||
topk: int,
|
||||
) -> CuTeDSLLaunchSpec[CompileKey]:
|
||||
compile_key = self.dispatch(
|
||||
topk=topk,
|
||||
num_candidates=gathered.shape[1],
|
||||
)
|
||||
return compile_key, (gathered, out)
|
||||
|
||||
|
||||
_PACK_DCP_TOPK_CANDIDATES_KERNEL = PackDCPTopkCandidatesKernel()
|
||||
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL = StableTopKFromGatheredCandidatesKernel()
|
||||
@@ -59,10 +59,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
|
||||
TritonW4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
|
||||
XPUW4A8IntLinearKernel,
|
||||
XPUwNa16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
|
||||
ZentorchWNA16LinearKernel,
|
||||
)
|
||||
@@ -91,9 +87,6 @@ from vllm.model_executor.kernels.linear.mxfp4.marlin import (
|
||||
from vllm.model_executor.kernels.linear.mxfp4.pytorch import (
|
||||
TorchMxFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp4.xpu import (
|
||||
XPUMxFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp6 import (
|
||||
MxFp6LinearKernel,
|
||||
MxFp6LinearLayerConfig,
|
||||
@@ -132,9 +125,6 @@ from vllm.model_executor.kernels.linear.mxfp8.pytorch import (
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.xpu import (
|
||||
XPUMxFp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.nvfp4 import (
|
||||
NvFp4LinearKernel,
|
||||
NvFp4LinearLayerConfig,
|
||||
@@ -228,11 +218,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
XPUFp8BlockScaledMMKernel,
|
||||
XPUW8A8FP8LinearKernel,
|
||||
XPUW8A16FP8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
|
||||
ZentorchInt8ScaledMMLinearKernel,
|
||||
)
|
||||
@@ -373,13 +358,6 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
EmulationMxfp6LinearKernel,
|
||||
EmulationMxfp4LinearKernel,
|
||||
},
|
||||
"xpu": {
|
||||
XPUW8A8FP8LinearKernel,
|
||||
XPUFp8BlockScaledMMKernel,
|
||||
},
|
||||
"xpu_woq": {
|
||||
XPUW8A16FP8LinearKernel,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -444,7 +422,6 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]
|
||||
HummingInt8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [AiterInt8ScaledMMLinearKernel, TritonInt8ScaledMMLinearKernel],
|
||||
PlatformEnum.XPU: [TritonInt8ScaledMMLinearKernel],
|
||||
}
|
||||
|
||||
# in priority/performance order (when available)
|
||||
@@ -473,12 +450,6 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUW8A16FP8LinearKernel,
|
||||
XPUW8A8FP8LinearKernel,
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@@ -504,11 +475,6 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
|
||||
CPUFp8W8A8BlockScaledMMKernel, # W8A8 preferred; falls back to W8A16 below
|
||||
CPUFp8BlockScaledMMKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUFp8BlockScaledMMKernel,
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
BlockWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
_POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
|
||||
@@ -522,9 +488,6 @@ _POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]
|
||||
PlatformEnum.CPU: [
|
||||
# To be added
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUW8A16FP8LinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
# in priority/performance order (when available)
|
||||
@@ -545,10 +508,6 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
|
||||
ConchLinearKernel,
|
||||
ExllamaLinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUW4A8IntLinearKernel,
|
||||
XPUwNa16LinearKernel,
|
||||
],
|
||||
PlatformEnum.CPU: [
|
||||
Dynamic4bitLinearKernel,
|
||||
ZentorchWNA16LinearKernel,
|
||||
@@ -574,11 +533,6 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
EmulationMxfp8LinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUMxFp8LinearKernel,
|
||||
TorchMxFp8LinearKernel,
|
||||
EmulationMxfp8LinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
_POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
|
||||
@@ -626,10 +580,6 @@ _POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = {
|
||||
AiterMxfp4LinearKernel,
|
||||
EmulationMxfp4LinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUMxFp4LinearKernel,
|
||||
TorchMxFp4LinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
# TODO make all kernels inherit from MMLinearKernel
|
||||
@@ -1348,7 +1298,6 @@ __all__ = [
|
||||
"MacheteLinearKernel",
|
||||
"MarlinLinearKernel",
|
||||
"TritonW4A16LinearKernel",
|
||||
"XPUW4A8IntLinearKernel",
|
||||
"XPUwNa16LinearKernel",
|
||||
"init_mxfp8_linear_kernel",
|
||||
"Mxfp8LinearKernel",
|
||||
@@ -1373,7 +1322,6 @@ __all__ = [
|
||||
"FlashInferCutlassMxfp8LinearKernel",
|
||||
"FlashInferTrtllmMxfp8LinearKernel",
|
||||
"MarlinMxfp8LinearKernel",
|
||||
"XPUMxFp8LinearKernel",
|
||||
"TorchMxFp8LinearKernel",
|
||||
"EmulationMxfp8LinearKernel",
|
||||
"CutlassNvFp4LinearKernel",
|
||||
|
||||
@@ -38,10 +38,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
|
||||
TritonW4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
|
||||
XPUW4A8IntLinearKernel,
|
||||
XPUwNa16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
|
||||
ZentorchWNA16LinearKernel,
|
||||
)
|
||||
@@ -60,7 +56,6 @@ __all__ = [
|
||||
"RDNA3W4A16LinearKernel",
|
||||
"RDNAHybridW4A16LinearKernel",
|
||||
"TritonW4A16LinearKernel",
|
||||
"XPUW4A8IntLinearKernel",
|
||||
"XPUwNa16LinearKernel",
|
||||
"ZentorchWNA16LinearKernel",
|
||||
]
|
||||
|
||||
@@ -1,218 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
|
||||
|
||||
_XPUWNA16_SUPPORTED_QUANT_TYPES = (scalar_types.uint4, scalar_types.uint4b8)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class XPUwNa16LinearKernel(MPLinearKernel):
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return -1
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUwNa16 only supported on XPU"
|
||||
|
||||
if c.act_type != torch.bfloat16 and c.act_type != torch.float16:
|
||||
return False, "XPUwNa16 only supports BF16/FP16 activations"
|
||||
|
||||
if c.weight_type not in _XPUWNA16_SUPPORTED_QUANT_TYPES:
|
||||
return (
|
||||
False,
|
||||
f"Quant type ({c.weight_type}) not supported by "
|
||||
"XPUwNa16, supported types are: "
|
||||
f"{_XPUWNA16_SUPPORTED_QUANT_TYPES}",
|
||||
)
|
||||
if c.group_size != -1 and c.group_size % 32 != 0:
|
||||
return (
|
||||
False,
|
||||
f"Group size ({c.group_size}) not supported by "
|
||||
"XPUwNa16, supported group sizes are multiples of 32",
|
||||
)
|
||||
|
||||
if c.partition_weight_shape[0] % 32 != 0:
|
||||
return (
|
||||
False,
|
||||
f"Input size ({c.partition_weight_shape[0]}) not supported by "
|
||||
"XPUwNa16, supported sizes are multiples of 32",
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
# Default name since marlin requires empty parameter for zp,
|
||||
# TODO: remove this requirement from marlin (allow optional tensors)
|
||||
if self.w_zp_name is None:
|
||||
self.w_zp_name = "w_zp"
|
||||
|
||||
need_transpose = False
|
||||
qweight_shape = getattr(layer, self.w_q_name).shape
|
||||
scale_shape = getattr(layer, self.w_s_name).shape
|
||||
# gptq marlin and compressed tensors wna16 expect different default
|
||||
# layouts for weight and scale, so we check the shapes to determine
|
||||
# if we need to transpose
|
||||
if qweight_shape[0] != scale_shape[0]:
|
||||
need_transpose = True
|
||||
|
||||
if need_transpose:
|
||||
getattr(layer, self.w_q_name).data = (
|
||||
getattr(layer, self.w_q_name).data.t().contiguous()
|
||||
)
|
||||
getattr(layer, self.w_s_name).data = getattr(layer, self.w_s_name).data
|
||||
else:
|
||||
getattr(layer, self.w_s_name).data = (
|
||||
getattr(layer, self.w_s_name).data.t().contiguous()
|
||||
)
|
||||
|
||||
if self.config.zero_points:
|
||||
# (FIXME): maybe zero points should also be transposed.
|
||||
getattr(layer, self.w_zp_name).data = (
|
||||
getattr(layer, self.w_zp_name).data.t().contiguous()
|
||||
)
|
||||
else:
|
||||
weight_zero_point = torch.Tensor([8]).to(torch.int8).to("xpu")
|
||||
setattr(
|
||||
layer, self.w_zp_name, Parameter(weight_zero_point, requires_grad=False)
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a16(
|
||||
reshaped_x,
|
||||
w_q.t(),
|
||||
bias if bias is not None else None,
|
||||
w_s,
|
||||
w_zp,
|
||||
self.config.group_size,
|
||||
None, # Retained by the external XPU op ABI.
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
class XPUW4A8IntLinearKernel(MPLinearKernel):
|
||||
"""XPU kernel for W4A8 integer quantization using oneDNN int4_gemm_w4a8.
|
||||
|
||||
Weights are symmetric group-quantized int4 packed as uint4.
|
||||
Activations are dynamically quantized per-token to symmetric int8.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return -1
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUW4A8Int only supported on XPU"
|
||||
if c.act_type not in (torch.bfloat16, torch.float16):
|
||||
return False, "XPUW4A8Int requires BF16/FP16 activations"
|
||||
if c.weight_type != scalar_types.int4:
|
||||
return (
|
||||
False,
|
||||
f"XPUW4A8Int requires int4 weights, got {c.weight_type}",
|
||||
)
|
||||
if c.zero_points:
|
||||
return False, "XPUW4A8Int only supports symmetric weight quantization"
|
||||
if c.group_size != -1 and c.group_size % 32 != 0:
|
||||
return (
|
||||
False,
|
||||
f"Group size ({c.group_size}) not supported by XPUW4A8Int, "
|
||||
"must be a multiple of 32",
|
||||
)
|
||||
in_size, out_size = c.partition_weight_shape
|
||||
if in_size % 8 != 0 or out_size % 8 != 0:
|
||||
return (
|
||||
False,
|
||||
f"in/out sizes ({in_size}, {out_size}) must be multiples of 8",
|
||||
)
|
||||
|
||||
if c.act_type != torch.float16:
|
||||
logger.warning_once(
|
||||
"XPUW4A8IntLinearKernel is running with model dtype %s, "
|
||||
"but int4_gemm_w4a8 produces float16 output. Recommend "
|
||||
"setting --dtype float16 for best performance.",
|
||||
c.act_type,
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
def _pack_int4_weight(self, w: torch.Tensor) -> torch.Tensor:
|
||||
# w is [N, K] int8 with values in [-8, 7]
|
||||
w_u4 = w.to(torch.int32) + 8 # shift to [0, 15]
|
||||
w_u4 = w_u4.reshape(w.shape[0], w.shape[1] // 8, 8) # [N, K/8, 8]
|
||||
shifts = torch.arange(0, 32, 4, dtype=torch.int32, device=w.device)
|
||||
packed = ((w_u4 & 0xF) << shifts[None, None, :]).sum(dim=2).to(torch.int32)
|
||||
return packed
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight_scale.data = layer.weight_scale.data.t().contiguous()
|
||||
|
||||
device = layer.weight_packed.device
|
||||
# TODO: support asymmetric quantization
|
||||
weight_zero_point = torch.tensor([8], dtype=torch.int8, device=device)
|
||||
layer.weight_zero_point = Parameter(weight_zero_point, requires_grad=False)
|
||||
|
||||
# weight_packed is [out, in] int8, signed int4 values in [-8, 7]
|
||||
w = layer.weight_packed.data # [out, in]
|
||||
|
||||
# TODO: implement asym case
|
||||
packed = self._pack_int4_weight(w) # [out, in/8] packed uint4
|
||||
|
||||
replace_parameter(
|
||||
layer,
|
||||
self.w_q_name,
|
||||
torch.nn.Parameter(packed, requires_grad=False),
|
||||
)
|
||||
|
||||
# Free the original unpacked int8 weight (still registered as "weight")
|
||||
# to avoid double-storing both int8 [N, K] and int32 [N, K/8] in memory.
|
||||
delattr(layer, "weight")
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = x.reshape(-1, x.shape[-1]) # [M, K]
|
||||
from vllm._xpu_ops import xpu_ops as ops
|
||||
|
||||
# TODO: static and asymmetric quantization case
|
||||
# Common code for CompressedTensorsW4A8Int does not read act symmetry data
|
||||
quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref(
|
||||
reshaped_x, True, 8
|
||||
)
|
||||
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a8(
|
||||
quant_x,
|
||||
x_scale,
|
||||
x_zero,
|
||||
layer.weight_packed.t(),
|
||||
layer.weight_scale,
|
||||
layer.weight_zero_point,
|
||||
self.config.group_size,
|
||||
None, # Retained by the external XPU op ABI.
|
||||
bias,
|
||||
)
|
||||
|
||||
return out.to(x.dtype)
|
||||
@@ -1,58 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
xpu_mxfp4_quantize as quant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
|
||||
|
||||
class XPUMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
"""MXFP4 W4A4 GEMM on XPU."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUMxFp4 only support on XPU"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if config.activation_quant_key != kMxfp4Dynamic:
|
||||
return False, "only supports MXFP4 dynamic activation"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
weight = layer.weight.view(torch.float4_e2m1fn_x2)
|
||||
replace_parameter(layer, "weight", weight.data.t())
|
||||
|
||||
weight_scale = layer.weight_scale.view(torch.float8_e8m0fnu)
|
||||
weight_scale = weight_scale.t().contiguous()
|
||||
replace_parameter(layer, "weight_scale", weight_scale.data)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
out_dtype = x.dtype
|
||||
x_fp4, x_blockscale = quant_mxfp4(x)
|
||||
return torch.ops._xpu_C.fp4_gemm(
|
||||
x_fp4,
|
||||
layer.weight,
|
||||
x_blockscale,
|
||||
layer.weight_scale,
|
||||
out_dtype,
|
||||
bias,
|
||||
)
|
||||
@@ -1,82 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
xpu_mxfp8_quantize as quant_mxfp8,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
|
||||
|
||||
class XPUMxFp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""MXFP8 W8A8 GEMM on XPU."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUMxFp8 only support on XPU"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Checkpoint scale is [N, K//32] (one E8M0 scale per 32 elements).
|
||||
# oneDNN fp8_gemm requires contiguous [K//32, N] layout. Store the
|
||||
# transposed contiguous buffer as a .t() view so that:
|
||||
# - dequantization consumers still see the checkpoint shape
|
||||
# - apply_weights recovers the oneDNN layout via .t() at zero cost
|
||||
weight_scale = layer.weight_scale.view(torch.float8_e8m0fnu)
|
||||
scale_kn = weight_scale.data.t().contiguous()
|
||||
replace_parameter(layer, "weight_scale", scale_kn.t())
|
||||
|
||||
if getattr(layer, "is_bmm", False):
|
||||
self._prepare_bmm_params(layer, scale_kn)
|
||||
|
||||
def _prepare_bmm_params(
|
||||
self, layer: torch.nn.Module, scale_kn: torch.Tensor
|
||||
) -> None:
|
||||
"""Precompute batched weight and scale for grouped fp8_bmm (e.g. wo_a).
|
||||
|
||||
Splits scale [K//32, N_total] into [G, K//32, N_per_group] and weight
|
||||
[N_total, K] into contiguous [G, K, N_per_group] for batch GEMM.
|
||||
"""
|
||||
batch = layer.bmm_batch_size
|
||||
k_blocks, n_blocks = scale_kn.shape
|
||||
layer.bmm_scale = (
|
||||
scale_kn.reshape(k_blocks, batch, n_blocks // batch)
|
||||
.permute(1, 0, 2)
|
||||
.contiguous()
|
||||
)
|
||||
w = layer.weight
|
||||
n_total, k = w.shape
|
||||
layer.bmm_weight = (
|
||||
w.reshape(batch, n_total // batch, k).permute(0, 2, 1).contiguous()
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
out_dtype = x.dtype
|
||||
x_fp8, x_scale = quant_mxfp8(x)
|
||||
# Weight is [N, K]; .t() gives a [K, N] view without copying.
|
||||
# Scale is stored as [N, K//32]; .t() recovers the contiguous
|
||||
# [K//32, N] buffer that oneDNN expects.
|
||||
return torch.ops._xpu_C.fp8_gemm(
|
||||
x_fp8,
|
||||
layer.weight.t(),
|
||||
out_dtype,
|
||||
x_scale,
|
||||
layer.weight_scale.t(),
|
||||
bias,
|
||||
)
|
||||
@@ -42,9 +42,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
XPUFp8BlockScaledMMKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
|
||||
ZentorchInt8ScaledMMLinearKernel,
|
||||
)
|
||||
@@ -73,5 +70,4 @@ __all__ = [
|
||||
"CPUFp8BlockScaledMMKernel",
|
||||
"CPUFp8PerTensorScaledMMLinearKernel",
|
||||
"CPUFP8W8A8ScaledMMLinearKernel",
|
||||
"XPUFp8BlockScaledMMKernel",
|
||||
]
|
||||
|
||||
@@ -1,284 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kFp8StaticTokenSym,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel
|
||||
from .ScaledMMLinearKernel import FP8ScaledMMLinearKernel, FP8ScaledMMLinearLayerConfig
|
||||
|
||||
|
||||
class XPUW8A8FP8LinearKernel(FP8ScaledMMLinearKernel):
|
||||
_SUPPORTED_ACT_QUANT_KEYS = {
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticTensorSym,
|
||||
kFp8StaticTokenSym,
|
||||
}
|
||||
_SUPPORTED_WEIGHT_QUANT_KEYS = {
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUW8A8FP8Linear only support on XPU"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if c.weight_quant_key not in cls._SUPPORTED_WEIGHT_QUANT_KEYS:
|
||||
return (
|
||||
False,
|
||||
"XPUW8A8FP8Linear only support per-channel and per-tensor quantization",
|
||||
)
|
||||
if c.activation_quant_key not in cls._SUPPORTED_ACT_QUANT_KEYS:
|
||||
return (
|
||||
False,
|
||||
"XPUW8A8FP8Linear only support per-tensor and per-token activation "
|
||||
"quantization",
|
||||
)
|
||||
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
|
||||
return False, "XPUW8A8FP8Linear only support FP8 weight dtype"
|
||||
if c.activation_quant_key.dtype not in {
|
||||
torch.float8_e5m2,
|
||||
torch.float8_e4m3fn,
|
||||
}:
|
||||
return False, "XPUW8A8FP8Linear only support FP8 activation dtype"
|
||||
return True, None
|
||||
|
||||
def __init__(
|
||||
self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str]
|
||||
) -> None:
|
||||
super().__init__(c, layer_param_names)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""Ensure weight is stored as C-contiguous [K, N] (KN layout).
|
||||
|
||||
Checkpoints store weight as [N, K]; fp8_gemm requires [K, N],
|
||||
C-contiguous. Three incoming layouts are possible:
|
||||
• [N, K] C-contiguous ← direct checkpoint → .t().contiguous()
|
||||
• [K, N] Fortran-order ← fp8.py's weight.t() → .contiguous()
|
||||
• [K, N] C-contiguous ← already correct → no-op
|
||||
|
||||
For square weights (K == N) the shape is ambiguous; contiguity is used
|
||||
as a proxy: C-contiguous ≡ checkpoint [N, K] (needs transpose);
|
||||
Fortran-order ≡ fp8.py already transposed (needs only contiguous).
|
||||
"""
|
||||
K = getattr(layer, "input_size_per_partition", self.config.weight_shape[1])
|
||||
N = getattr(layer, "output_size_per_partition", self.config.weight_shape[0])
|
||||
w = layer.weight
|
||||
|
||||
if w.shape not in {(K, N), (N, K)}:
|
||||
raise ValueError(
|
||||
f"XPUFP8ScaledMM expects weight shape ({K},{N}) or ({N},{K}), "
|
||||
f"but got {tuple(w.shape)}"
|
||||
)
|
||||
|
||||
needs_transpose = w.shape == (N, K) if K != N else w.is_contiguous()
|
||||
layer_weight = w.t() if needs_transpose else w
|
||||
replace_parameter(layer, "weight", layer_weight)
|
||||
ws = layer.weight_scale
|
||||
if ws.numel() == 1:
|
||||
replace_parameter(layer, "weight_scale", ws.reshape(1))
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
# B is C-contiguous [K, N] from process_weights_after_loading.
|
||||
# fp8_gemm routes on scale dtype (float32) and numel:
|
||||
# As [1] → per-tensor (numel==1 branch)
|
||||
# As [M,1] → per-token (group={1,K} branch, broadcast across K)
|
||||
# Bs [1] → per-tensor
|
||||
# Bs [N] → per-channel (mask=bit1 branch)
|
||||
# No shape manipulation needed here.
|
||||
output = torch.ops._xpu_C.fp8_gemm(A, B, out_dtype, As, Bs, bias)
|
||||
return output.view(*output_shape)
|
||||
|
||||
|
||||
class XPUW8A16FP8LinearKernel(FP8ScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUW8A16FP8Linear only support on XPU"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if c.weight_quant_key not in {kFp8StaticChannelSym, kFp8StaticTensorSym}:
|
||||
return (
|
||||
False,
|
||||
"XPUW8A16FP8Linear only support per-channel and per-tensor "
|
||||
"quantization",
|
||||
)
|
||||
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
|
||||
return False, "XPUW8A16FP8Linear only support FP8 weight dtype"
|
||||
return True, None
|
||||
|
||||
def __init__(
|
||||
self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str]
|
||||
) -> None:
|
||||
assert self.can_implement(c)[0]
|
||||
assert self.is_supported()[0]
|
||||
self.config = c
|
||||
self.layer_param_names = layer_param_names
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# fp8_gemm_w8a16 expects weight in [in, out] layout.
|
||||
# Transpose if weight is still in [out, in] layout.
|
||||
# For square matrices, use contiguity as tie-breaker:
|
||||
# checkpoint weights are contiguous, .t() views are not.
|
||||
weight = layer.weight
|
||||
out_features, in_features = self.config.weight_shape
|
||||
|
||||
if weight.shape == (out_features, in_features) and (
|
||||
in_features != out_features or weight.is_contiguous()
|
||||
):
|
||||
replace_parameter(layer, "weight", weight.data.t())
|
||||
# else: already in [in, out] layout — no-op
|
||||
|
||||
weight_scale = layer.weight_scale.t().contiguous()
|
||||
replace_parameter(layer, "weight_scale", weight_scale.data)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
weight = layer.weight
|
||||
weight_scale = layer.weight_scale
|
||||
return torch.ops._xpu_C.fp8_gemm_w8a16(x, weight, weight_scale, bias)
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
|
||||
|
||||
class XPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUFp8BlockScaledMM only support on XPU"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
super().process_weights_after_loading(layer)
|
||||
scale_attr = (
|
||||
"weight_scale_inv" if hasattr(layer, "weight_scale_inv") else "weight_scale"
|
||||
)
|
||||
scale = getattr(layer, scale_attr)
|
||||
|
||||
# Ragged N (N % block_n != 0): oneDNN needs n_blocks to divide N.
|
||||
# Weight untouched; only repeat scale rows to a finer N-group gn that
|
||||
# divides both N and block_n (gn = gcd(N, block_n)):
|
||||
# scale [ceil(N/block_n), K/block_k] --> [N/gn, K/block_k]
|
||||
# oneDNN only accepts gn that is a multiple of 16, and gcd(N, block_n)
|
||||
# is a power of two (block_n=128), so gn must be >= 16. No-op when
|
||||
# N % block_n == 0.
|
||||
block_n, block_k = self.weight_group_shape
|
||||
N, K = layer.weight.shape
|
||||
if N % block_n != 0:
|
||||
gn = math.gcd(N, block_n)
|
||||
assert gn % 16 == 0, (
|
||||
f"XPU block-scaled FP8: N ({N}) yields group width {gn}, but "
|
||||
f"oneDNN only supports multiples of 16; this weight shape is "
|
||||
f"unsupported."
|
||||
)
|
||||
col_start = torch.arange(N // gn, device=scale.device) * gn
|
||||
src_idx = torch.div(col_start, block_n, rounding_mode="floor")
|
||||
scale = scale.index_select(0, src_idx).contiguous()
|
||||
|
||||
# Ragged K needs the runtime activation scale expanded too, which we
|
||||
# don't handle; DeepSeek/GLM keep K block-aligned, so fail loudly.
|
||||
assert K % block_k == 0, (
|
||||
f"XPU block-scaled FP8 requires K ({K}) to be a multiple of the "
|
||||
f"weight block size ({block_k}); ragged-K weights are unsupported."
|
||||
)
|
||||
|
||||
# Checkpoint scale is [n_blocks, k_blocks] (one value per block tile).
|
||||
# oneDNN fp8_gemm requires contiguous [k_blocks, n_blocks] layout.
|
||||
# We store the transposed contiguous buffer as a .t() view so that:
|
||||
# - MLA's scaled_dequantize still sees [n_blocks, k_blocks] shape
|
||||
# - apply_block_scaled_mm recovers the contiguous buffer via .t()
|
||||
scale_kn = scale.data.t().contiguous() # [k_blocks, n_blocks]
|
||||
replace_parameter(layer, scale_attr, scale_kn.t()) # view: [n_blocks, k_blocks]
|
||||
|
||||
if getattr(layer, "is_bmm", False):
|
||||
self._prepare_bmm_params(layer, scale_kn)
|
||||
|
||||
def _prepare_bmm_params(
|
||||
self, layer: torch.nn.Module, scale_kn: torch.Tensor
|
||||
) -> None:
|
||||
"""Precompute batched weight and scale for grouped fp8_bmm (e.g. wo_a).
|
||||
|
||||
Splits scale [k_blocks, n_blocks] into [G, k_blocks, n_blocks_per_group]
|
||||
and weight [N_total, K] into [G, K, N_per_group] for batch GEMM.
|
||||
"""
|
||||
batch = layer.bmm_batch_size
|
||||
k_blocks, n_blocks = scale_kn.shape
|
||||
layer.bmm_scale = (
|
||||
scale_kn.reshape(k_blocks, batch, n_blocks // batch)
|
||||
.permute(1, 0, 2)
|
||||
.contiguous()
|
||||
)
|
||||
w = layer.weight
|
||||
N_total, K = w.shape
|
||||
layer.bmm_weight = w.reshape(batch, N_total // batch, K).permute(
|
||||
0, 2, 1
|
||||
) # [G, K, N_per_group]
|
||||
|
||||
def apply_block_scaled_mm(
|
||||
self,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# B is [N, K]; .t() gives [K, N] view (no copy).
|
||||
# Bs is stored as [n_blocks, k_blocks] view; .t() recovers the
|
||||
# contiguous [k_blocks, n_blocks] buffer that oneDNN expects.
|
||||
return torch.ops._xpu_C.fp8_gemm(
|
||||
A,
|
||||
B.t(),
|
||||
self.config.out_dtype,
|
||||
As,
|
||||
Bs.t(),
|
||||
torch.Tensor(),
|
||||
)
|
||||
@@ -1,36 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .aiter import *
|
||||
from .cpu import *
|
||||
from .tilelang import *
|
||||
from .torch import *
|
||||
from .triton import *
|
||||
|
||||
__all__ = [
|
||||
"mhc_pre_cuda",
|
||||
"mhc_post_cuda",
|
||||
"mhc_fused_post_pre_cuda",
|
||||
"hc_head_fused_kernel_cuda",
|
||||
"mhc_pre_aiter",
|
||||
"mhc_pre_delayed_aiter",
|
||||
"mhc_fused_post_pre_delayed_rms_norm_aiter",
|
||||
"mhc_post_aiter",
|
||||
"mhc_fused_post_pre_aiter",
|
||||
"hc_head_fused_aiter",
|
||||
"mhc_pre_cpu",
|
||||
"mhc_post_cpu",
|
||||
"hc_head_fused_cpu",
|
||||
"mhc_pre_tilelang",
|
||||
"mhc_post_tilelang",
|
||||
"mhc_fused_post_pre_tilelang",
|
||||
"hc_head_fused_tilelang",
|
||||
"mhc_pre_torch",
|
||||
"mhc_post_torch",
|
||||
"mhc_fused_post_pre_torch",
|
||||
"hc_head_fused_torch",
|
||||
"mhc_pre_mix_triton",
|
||||
"mhc_pre_triton",
|
||||
"mhc_post_triton",
|
||||
"mhc_fused_post_pre_triton",
|
||||
"hc_head_fused_triton",
|
||||
]
|
||||
@@ -1,428 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
def mhc_pre_aiter(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
|
||||
hc_scale: shape (3,), dtype torch.float32
|
||||
hc_base: shape (hc_mult3,), dtype torch.float32
|
||||
rms_eps: RMS normalization epsilon
|
||||
hc_pre_eps: pre-mix epsilon
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
n_splits: split-k factor;
|
||||
norm_weight: optional RMSNorm weight fused into the pre kernel
|
||||
norm_eps: epsilon for the fused RMSNorm when norm_weight is set
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
|
||||
"""
|
||||
hidden_size = residual.shape[-1]
|
||||
assert hidden_size % 256 == 0
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_pre(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_pre_aiter_fake(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
# Create empty tensors with correct shapes for meta device / shape inference
|
||||
post_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
1,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input = torch.empty(
|
||||
*outer_shape,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
return post_mix, comb_mix, layer_input
|
||||
|
||||
|
||||
def mhc_pre_delayed_aiter(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
sublayer_out: torch.Tensor | None = None,
|
||||
post_layer_mix: torch.Tensor | None = None,
|
||||
comb_res_mix: torch.Tensor | None = None,
|
||||
residual_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""MHC pre with the pre-mix carried in from the previous sublayer.
|
||||
|
||||
Matches ``mhc_pre_delayed_torch``: the stream collapse uses *pre_mix*
|
||||
rather than the gate computed here, and that gate is returned as the
|
||||
pre-mix for the next sublayer seam.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
|
||||
hc_scale: shape (3,), dtype torch.float32
|
||||
hc_base: shape (hc_mult3,), dtype torch.float32
|
||||
rms_eps: RMS normalization epsilon
|
||||
hc_pre_eps: pre-mix epsilon
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
pre_mix: shape (..., hc_mult) from the previous sublayer, or None at
|
||||
model entry to select residual stream zero.
|
||||
sublayer_out: attention or FFN output, shape (..., hidden_size). When
|
||||
given with the two mixes below, the preceding post block is
|
||||
applied here so AITER can fold it into the pre projection.
|
||||
post_layer_mix: shape (..., hc_mult, 1), post gate for that block.
|
||||
comb_res_mix: shape (..., hc_mult, hc_mult), residual comb for it.
|
||||
residual_out: shape (..., hc_mult, hidden_size), written with the post
|
||||
block's new residual. Required exactly when the post is folded in.
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult, 1), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
next_pre_mix: shape (..., hc_mult), dtype torch.float32
|
||||
|
||||
"""
|
||||
hidden_size = residual.shape[-1]
|
||||
assert hidden_size % 256 == 0
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_pre_delayed(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
sublayer_out,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
residual_out,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_pre_delayed_aiter_fake(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
sublayer_out: torch.Tensor | None = None,
|
||||
post_layer_mix: torch.Tensor | None = None,
|
||||
comb_res_mix: torch.Tensor | None = None,
|
||||
residual_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
return (
|
||||
torch.empty(
|
||||
*outer_shape, hc_mult, 1, dtype=torch.float32, device=residual.device
|
||||
),
|
||||
torch.empty(
|
||||
*outer_shape, hc_mult, hc_mult, dtype=torch.float32, device=residual.device
|
||||
),
|
||||
torch.empty(
|
||||
*outer_shape, hidden_size, dtype=torch.bfloat16, device=residual.device
|
||||
),
|
||||
torch.empty(*outer_shape, hc_mult, dtype=torch.float32, device=residual.device),
|
||||
)
|
||||
|
||||
|
||||
def mhc_fused_post_pre_delayed_rms_norm_aiter(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None,
|
||||
sublayer_out: torch.Tensor | None,
|
||||
post_layer_mix: torch.Tensor | None,
|
||||
comb_res_mix: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
norm_eps: float,
|
||||
residual_out: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
""":func:`mhc_pre_delayed_aiter` with the RMSNorm of the collapse folded in:
|
||||
``layer_input`` comes back normalised with ``norm_weight`` / ``norm_eps``."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_fused_post_pre_delayed_rms_norm(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
sublayer_out,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
residual_out,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_fused_post_pre_delayed_rms_norm_aiter_fake(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None,
|
||||
sublayer_out: torch.Tensor | None,
|
||||
post_layer_mix: torch.Tensor | None,
|
||||
comb_res_mix: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
norm_eps: float,
|
||||
residual_out: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return _mhc_pre_delayed_aiter_fake(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
|
||||
def mhc_post_aiter(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
hidden_size = residual.shape[-1]
|
||||
|
||||
assert hidden_size % 256 == 0
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_post(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_post_aiter_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(residual)
|
||||
|
||||
|
||||
def mhc_fused_post_pre_aiter(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
tile_n: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Fused mHC post + next mHC pre on ROCm via AITER.
|
||||
|
||||
Returns residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur.
|
||||
"""
|
||||
hidden_size = residual.shape[-1]
|
||||
assert hidden_size % 256 == 0
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_fused_post_pre(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_fused_post_pre_aiter_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
tile_n: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
post_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
1,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input = torch.empty(
|
||||
*outer_shape,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
return torch.empty_like(residual), post_mix, comb_mix, layer_input
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre_aiter",
|
||||
op_func=mhc_pre_aiter,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_pre_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre_delayed_aiter",
|
||||
op_func=mhc_pre_delayed_aiter,
|
||||
mutates_args=["residual_out"],
|
||||
fake_impl=_mhc_pre_delayed_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_fused_post_pre_delayed_rms_norm_aiter",
|
||||
op_func=mhc_fused_post_pre_delayed_rms_norm_aiter,
|
||||
mutates_args=["residual_out"],
|
||||
fake_impl=_mhc_fused_post_pre_delayed_rms_norm_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_post_aiter",
|
||||
op_func=mhc_post_aiter,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_post_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_fused_post_pre_aiter",
|
||||
op_func=mhc_fused_post_pre_aiter,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_fused_post_pre_aiter_fake,
|
||||
)
|
||||
@@ -1,96 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
|
||||
def mhc_pre_cpu(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""CPU-ported mHC pre block (see `mhc_pre_torch` for the eager reference).
|
||||
|
||||
The ported kernel (`hc_pre_fused_cpu`) only exposes one merged `hc_eps`
|
||||
(used for both `hc_pre_eps`/`hc_sinkhorn_eps`) and hardcodes the post-mix
|
||||
multiplier to 2.0 -- true of every real call site in
|
||||
`models/deepseek_v4/cpu/model.py`, so this is not a capability loss here.
|
||||
"""
|
||||
assert n_splits == 1, "mhc_pre_cpu does not support n_splits != 1"
|
||||
assert hc_pre_eps == hc_sinkhorn_eps, (
|
||||
"mhc_pre_cpu requires hc_pre_eps == hc_sinkhorn_eps (single merged hc_eps)"
|
||||
)
|
||||
assert hc_post_mult_value == 2.0, "mhc_pre_cpu hardcodes the post-mix multiplier"
|
||||
|
||||
hc_mult, hidden_size = residual.shape[-2:]
|
||||
outer_shape = residual.shape[:-2]
|
||||
x_flat = residual.reshape(-1, hc_mult, hidden_size)
|
||||
|
||||
layer_input, post, comb = ops.hc_pre_fused_cpu(
|
||||
x_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
hc_mult,
|
||||
sinkhorn_repeat,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
)
|
||||
|
||||
post_mix = post.view(*outer_shape, hc_mult, 1)
|
||||
comb_mix = comb.view(*outer_shape, hc_mult, hc_mult)
|
||||
layer_input = layer_input.view(*outer_shape, hidden_size)
|
||||
return post_mix, comb_mix, layer_input
|
||||
|
||||
|
||||
def mhc_post_cpu(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""CPU-ported mHC post block (see `mhc_post_torch` for the eager reference)."""
|
||||
hc_mult, hidden_size = residual.shape[-2:]
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
x_flat = x.reshape(-1, hidden_size)
|
||||
residual_flat = residual.reshape(-1, hc_mult, hidden_size)
|
||||
post_flat = post_layer_mix.reshape(-1, hc_mult).float()
|
||||
comb_flat = comb_res_mix.reshape(-1, hc_mult, hc_mult).float()
|
||||
|
||||
out = ops.hc_post_fused_cpu(x_flat, residual_flat, post_flat, comb_flat)
|
||||
return out.view(*outer_shape, hc_mult, hidden_size)
|
||||
|
||||
|
||||
def hc_head_fused_cpu(
|
||||
hidden_states: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
hc_eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""CPU-ported HC head reduction (see `test_hc_head_cpu` in
|
||||
tests/kernels/mhc/test_mhc_kernels.py for the eager reference this is tested
|
||||
against).
|
||||
|
||||
The ported kernel's C++ signature takes `(hc_eps, norm_eps)` -- the
|
||||
opposite order from this wrapper's `(rms_norm_eps, hc_eps)`, which matches
|
||||
the real call site in `models/deepseek_v4/cpu/model.py`.
|
||||
"""
|
||||
hc_mult, hidden_size = hidden_states.shape[-2:]
|
||||
outer_shape = hidden_states.shape[:-2]
|
||||
hs_flat = hidden_states.reshape(-1, hc_mult, hidden_size)
|
||||
|
||||
out = ops.hc_head_fused_cpu(hs_flat, hc_fn, hc_scale, hc_base, hc_eps, rms_norm_eps)
|
||||
return out.view(*outer_shape, hidden_size)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,147 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
|
||||
def mhc_pre_delayed_torch(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
x: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Reference for mHC pre using coefficients from the previous sublayer."""
|
||||
hc_mult = residual.shape[1]
|
||||
x = (residual.flatten(1) if x is None else x).float()
|
||||
mixes = (x @ fn.t()) * torch.rsqrt(x.square().mean(-1, keepdim=True) + rms_eps)
|
||||
pre = (
|
||||
torch.sigmoid(mixes[:, :hc_mult] * hc_scale[0] + hc_base[:hc_mult]) + hc_pre_eps
|
||||
)
|
||||
post = (
|
||||
torch.sigmoid(
|
||||
mixes[:, hc_mult : 2 * hc_mult] * hc_scale[1]
|
||||
+ hc_base[hc_mult : 2 * hc_mult]
|
||||
)
|
||||
* hc_post_mult_value
|
||||
)
|
||||
comb = mixes[:, 2 * hc_mult :].view(-1, hc_mult, hc_mult) * hc_scale[2]
|
||||
comb = comb + hc_base[2 * hc_mult :].view(1, hc_mult, hc_mult)
|
||||
comb = torch.softmax(comb, dim=-1) + hc_sinkhorn_eps
|
||||
comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
|
||||
for _ in range(sinkhorn_repeat - 1):
|
||||
comb = comb / (comb.sum(dim=-1, keepdim=True) + hc_sinkhorn_eps)
|
||||
comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
|
||||
layer_input = (
|
||||
residual[:, 0]
|
||||
if pre_mix is None
|
||||
else (pre_mix.unsqueeze(-1) * residual.float()).sum(dim=1).to(residual.dtype)
|
||||
)
|
||||
return post.unsqueeze(-1), comb, layer_input, pre
|
||||
|
||||
|
||||
def mhc_pre_torch(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
|
||||
hc_scale: shape (3,), dtype torch.float32
|
||||
hc_base: shape (hc_mult3,), dtype torch.float32
|
||||
rms_eps: RMS normalization epsilon
|
||||
hc_pre_eps: pre-mix epsilon
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
n_splits: split-k factor;
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
|
||||
"""
|
||||
# Validate shapes
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert fn.dtype == torch.float32
|
||||
assert hc_scale.dtype == torch.float32
|
||||
assert hc_base.dtype == torch.float32
|
||||
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult2 = hc_mult * hc_mult
|
||||
hc_mult3 = hc_mult * 2 + hc_mult2
|
||||
|
||||
hc_hidden_size = hc_mult * hidden_size
|
||||
assert fn.shape[0] == hc_mult3
|
||||
assert fn.shape[1] == hc_hidden_size
|
||||
assert hc_scale.shape == (3,)
|
||||
assert hc_base.shape == (hc_mult3,)
|
||||
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
fn_flat = fn
|
||||
|
||||
x = residual_flat.view(num_tokens, hc_mult * hidden_size).to(torch.float32)
|
||||
mixes = torch.matmul(x, fn_flat.t())
|
||||
sqrsum = x.square().sum(dim=-1, keepdim=True)
|
||||
mixes = mixes * torch.rsqrt(sqrsum / (hc_mult * hidden_size) + rms_eps)
|
||||
|
||||
pre_logits = mixes[:, :hc_mult] * hc_scale[0] + hc_base[:hc_mult]
|
||||
pre_mix = torch.sigmoid(pre_logits) + hc_pre_eps
|
||||
|
||||
post_logits = (
|
||||
mixes[:, hc_mult : 2 * hc_mult] * hc_scale[1] + hc_base[hc_mult : 2 * hc_mult]
|
||||
)
|
||||
post_mix = torch.sigmoid(post_logits) * hc_post_mult_value
|
||||
|
||||
comb_logits = mixes[:, 2 * hc_mult :].view(num_tokens, hc_mult, hc_mult) * hc_scale[
|
||||
2
|
||||
] + hc_base[2 * hc_mult :].view(1, hc_mult, hc_mult)
|
||||
comb_mix = torch.softmax(comb_logits, dim=-1) + hc_sinkhorn_eps
|
||||
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
|
||||
for _ in range(sinkhorn_repeat - 1):
|
||||
comb_mix = comb_mix / (comb_mix.sum(dim=-1, keepdim=True) + hc_sinkhorn_eps)
|
||||
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
|
||||
|
||||
layer_input = torch.sum(
|
||||
pre_mix.unsqueeze(-1) * residual_flat.to(torch.float32), dim=1
|
||||
).to(torch.bfloat16)
|
||||
return (
|
||||
post_mix.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix.view(*outer_shape, hc_mult, hc_mult),
|
||||
layer_input.view(*outer_shape, hidden_size),
|
||||
)
|
||||
|
||||
|
||||
def mhc_post_torch(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
mixed_residual = torch.einsum(
|
||||
"...ij,...ih->...jh",
|
||||
comb_res_mix.to(torch.float32),
|
||||
residual.to(torch.float32),
|
||||
)
|
||||
post_term = post_layer_mix.to(torch.float32) * x.unsqueeze(-2).to(torch.float32)
|
||||
return (mixed_residual + post_term).to(residual.dtype)
|
||||
@@ -1,349 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import Tensor
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rmsnorm_nw_kernel(
|
||||
x_ptr,
|
||||
out_ptr,
|
||||
stride_row,
|
||||
D,
|
||||
eps,
|
||||
RBLOCK: tl.constexpr,
|
||||
):
|
||||
"""Weight-free RMSNorm Triton kernel: out = x * rsqrt(mean(x², -1) + eps)."""
|
||||
row = tl.program_id(0)
|
||||
cols = tl.arange(0, RBLOCK)
|
||||
mask = cols < D
|
||||
|
||||
x = tl.load(
|
||||
x_ptr + row * stride_row + cols,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
eviction_policy="evict_first",
|
||||
).to(tl.float32)
|
||||
|
||||
var = tl.sum(x * x, 0) / D
|
||||
rstd = tl.rsqrt(var + eps)
|
||||
|
||||
out = (x * rstd).to(out_ptr.dtype.element_ty)
|
||||
tl.store(out_ptr + row * D + cols, out, mask=mask, eviction_policy="evict_first")
|
||||
|
||||
|
||||
def rmsnorm_nw(x: Tensor, eps: float) -> Tensor:
|
||||
"""Weight-free RMSNorm over the last dimension.
|
||||
|
||||
Treats *x* as ``[num_rows, D]`` where ``num_rows = product(shape[:-1])``.
|
||||
Returns a contiguous tensor with the same shape and dtype as *x*.
|
||||
"""
|
||||
orig_shape = x.shape
|
||||
D = orig_shape[-1]
|
||||
x_2d = x.reshape(-1, D)
|
||||
num_rows = x_2d.shape[0]
|
||||
|
||||
out = torch.empty_like(x_2d)
|
||||
RBLOCK = triton.next_power_of_2(D)
|
||||
|
||||
_rmsnorm_nw_kernel[(num_rows,)](
|
||||
x_2d,
|
||||
out,
|
||||
x_2d.stride(0),
|
||||
D,
|
||||
eps,
|
||||
RBLOCK=RBLOCK,
|
||||
num_warps=1 if RBLOCK <= 512 else (4 if RBLOCK <= 4096 else 8),
|
||||
)
|
||||
return out.view(orig_shape)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_head_reduce_store_kernel(
|
||||
pre_ptr,
|
||||
x_ptr,
|
||||
out_ptr,
|
||||
hidden_size: tl.constexpr,
|
||||
hc_mult: tl.constexpr,
|
||||
pre_stride_t: tl.constexpr,
|
||||
pre_stride_m: tl.constexpr,
|
||||
x_stride_t: tl.constexpr,
|
||||
x_stride_m: tl.constexpr,
|
||||
x_stride_h: tl.constexpr,
|
||||
out_stride_t: tl.constexpr,
|
||||
out_stride_h: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
block_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
mask = offsets < hidden_size
|
||||
|
||||
acc = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||||
for mix_idx in tl.static_range(0, hc_mult):
|
||||
pre = tl.load(pre_ptr + token_idx * pre_stride_t + mix_idx * pre_stride_m).to(
|
||||
tl.float32
|
||||
)
|
||||
x = tl.load(
|
||||
x_ptr
|
||||
+ token_idx * x_stride_t
|
||||
+ mix_idx * x_stride_m
|
||||
+ offsets * x_stride_h,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
acc += pre * x
|
||||
|
||||
tl.store(
|
||||
out_ptr + token_idx * out_stride_t + offsets * out_stride_h,
|
||||
acc,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
def hc_collapse_triton(x: Tensor, pre_mix: Tensor) -> Tensor:
|
||||
"""Collapse BF16 residual streams with FP32 pre-mix coefficients."""
|
||||
assert x.ndim == 3 and x.dtype == torch.bfloat16
|
||||
num_tokens, hc_mult, hidden_size = x.shape
|
||||
assert pre_mix.shape == (num_tokens, hc_mult)
|
||||
assert pre_mix.dtype == torch.float32
|
||||
out = torch.empty(num_tokens, hidden_size, dtype=x.dtype, device=x.device)
|
||||
if num_tokens == 0:
|
||||
return out
|
||||
|
||||
block_h = 1024
|
||||
_hc_head_reduce_store_kernel[(num_tokens, triton.cdiv(hidden_size, block_h))](
|
||||
pre_mix,
|
||||
x,
|
||||
out,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
pre_mix.stride(0),
|
||||
pre_mix.stride(1),
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
x.stride(2),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
BLOCK_H=block_h,
|
||||
num_warps=4,
|
||||
# Preserve the separate FP32 multiply and sum in the Torch reference.
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _hc_collapse_triton_fake(x: Tensor, pre_mix: Tensor) -> Tensor:
|
||||
return torch.empty(x.shape[0], x.shape[2], dtype=x.dtype, device=x.device)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mhc_pre_mix_kernel(
|
||||
gemm_ptr,
|
||||
sqrsum_ptr,
|
||||
hc_scale_ptr,
|
||||
hc_base_ptr,
|
||||
out_ptr,
|
||||
splitk,
|
||||
hc_mult: tl.constexpr,
|
||||
gemm_stride_k,
|
||||
gemm_stride_t,
|
||||
gemm_stride_j,
|
||||
sqrsum_stride_k,
|
||||
sqrsum_stride_t,
|
||||
out_stride_t,
|
||||
out_stride_j,
|
||||
inv_hc_hidden,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
SPLITK_BLOCK: tl.constexpr,
|
||||
HC_BLOCK: tl.constexpr,
|
||||
):
|
||||
"""Recover the pre-mix gate from a split-k mHC pre GEMM output."""
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
ks = tl.arange(0, SPLITK_BLOCK)
|
||||
js = tl.arange(0, HC_BLOCK)
|
||||
kmask = ks < splitk
|
||||
jmask = js < hc_mult
|
||||
|
||||
# Only the first hc_mult GEMM columns feed the pre gate.
|
||||
gemm = tl.load(
|
||||
gemm_ptr
|
||||
+ ks[:, None] * gemm_stride_k
|
||||
+ token_idx * gemm_stride_t
|
||||
+ js[None, :] * gemm_stride_j,
|
||||
mask=kmask[:, None] & jmask[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
mixes = tl.sum(gemm, 0)
|
||||
|
||||
sqrsum = tl.load(
|
||||
sqrsum_ptr + ks * sqrsum_stride_k + token_idx * sqrsum_stride_t,
|
||||
mask=kmask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
rstd = tl.rsqrt(tl.sum(sqrsum, 0) * inv_hc_hidden + rms_eps)
|
||||
|
||||
scale = tl.load(hc_scale_ptr).to(tl.float32)
|
||||
base = tl.load(hc_base_ptr + js, mask=jmask, other=0.0).to(tl.float32)
|
||||
|
||||
pre = tl.sigmoid(mixes * rstd * scale + base) + hc_pre_eps
|
||||
tl.store(out_ptr + token_idx * out_stride_t + js * out_stride_j, pre, mask=jmask)
|
||||
|
||||
|
||||
def mhc_pre_mix_triton(
|
||||
gemm_out: Tensor,
|
||||
sqrsum: Tensor,
|
||||
hc_scale: Tensor,
|
||||
hc_base: Tensor,
|
||||
hc_mult: int,
|
||||
hc_hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
) -> Tensor:
|
||||
"""Pre-mix gate for the delayed mHC pre, from AITER's split-k GEMM output.
|
||||
|
||||
AITER's ``mhc_pre_big_fuse`` consumes the unreduced ``[splitk, tokens,
|
||||
hc_mult3]`` GEMM output and the matching row square-sums, but only returns
|
||||
the post and comb gates. The delayed formulation also needs the pre gate,
|
||||
to carry into the next sublayer seam. It is the same slice of the same
|
||||
numbers, so recover it here rather than repeating the projection.
|
||||
"""
|
||||
assert gemm_out.ndim == 3 and gemm_out.dtype == torch.float32
|
||||
assert sqrsum.ndim == 2 and sqrsum.dtype == torch.float32
|
||||
splitk, num_tokens = gemm_out.shape[0], gemm_out.shape[1]
|
||||
assert sqrsum.shape == (splitk, num_tokens)
|
||||
|
||||
out = torch.empty(num_tokens, hc_mult, dtype=torch.float32, device=gemm_out.device)
|
||||
if num_tokens == 0:
|
||||
return out
|
||||
|
||||
_mhc_pre_mix_kernel[(num_tokens,)](
|
||||
gemm_out,
|
||||
sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
splitk,
|
||||
hc_mult,
|
||||
gemm_out.stride(0),
|
||||
gemm_out.stride(1),
|
||||
gemm_out.stride(2),
|
||||
sqrsum.stride(0),
|
||||
sqrsum.stride(1),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
1.0 / hc_hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
SPLITK_BLOCK=triton.next_power_of_2(splitk),
|
||||
HC_BLOCK=triton.next_power_of_2(hc_mult),
|
||||
num_warps=1,
|
||||
# Match the separate FP32 multiply and add in the Torch reference.
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _mhc_pre_mix_triton_fake(
|
||||
gemm_out: Tensor,
|
||||
sqrsum: Tensor,
|
||||
hc_scale: Tensor,
|
||||
hc_base: Tensor,
|
||||
hc_mult: int,
|
||||
hc_hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
) -> Tensor:
|
||||
return torch.empty(
|
||||
gemm_out.shape[1], hc_mult, dtype=torch.float32, device=gemm_out.device
|
||||
)
|
||||
|
||||
|
||||
def hc_head_reduce_triton_kernel(
|
||||
x: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
norm_eps: float,
|
||||
hc_eps: float,
|
||||
) -> None:
|
||||
x_flat = x.flatten(-2)
|
||||
x_normed = rmsnorm_nw(x_flat, norm_eps)
|
||||
mixes = F.linear(x_normed.float(), hc_fn)
|
||||
pre = torch.sigmoid(mixes * hc_scale + hc_base) + hc_eps
|
||||
|
||||
hidden_size = x.shape[-1]
|
||||
hc_mult = x.shape[-2]
|
||||
block_h = 1024
|
||||
_hc_head_reduce_store_kernel[(x.shape[0], (hidden_size + block_h - 1) // block_h)](
|
||||
pre,
|
||||
x,
|
||||
out,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
pre.stride(0),
|
||||
pre.stride(1),
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
x.stride(2),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
BLOCK_H=block_h,
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
|
||||
def _hc_head_triton(
|
||||
hs_flat: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
hc_mult: int,
|
||||
) -> None:
|
||||
"""Fill pre-allocated `out` (T, H) in-place with the hc_head result."""
|
||||
if hs_flat.shape[0] == 0:
|
||||
return
|
||||
|
||||
hc_head_reduce_triton_kernel(
|
||||
hs_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
rms_eps,
|
||||
hc_eps,
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="hc_head_triton",
|
||||
op_func=_hc_head_triton,
|
||||
mutates_args=["out"],
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="hc_collapse_triton",
|
||||
op_func=hc_collapse_triton,
|
||||
mutates_args=[],
|
||||
fake_impl=_hc_collapse_triton_fake,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre_mix_triton",
|
||||
op_func=mhc_pre_mix_triton,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_pre_mix_triton_fake,
|
||||
)
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
compute_num_split,
|
||||
mhc_pre_big_fuse_with_norm_tilelang,
|
||||
)
|
||||
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
|
||||
def compute_mhc_pre_num_splits(input_size: int, num_tokens: int) -> int:
|
||||
splits = compute_num_split(64, input_size, cdiv(num_tokens, 64))
|
||||
# Bound both GEMM and fused-normalization specializations during startup.
|
||||
return 1 if splits == 1 else 4 if splits <= 4 else 16
|
||||
|
||||
|
||||
class MHCPreNormKernel(VllmJitKernel["MHCPreNormKernel.CompileKey"]):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
hidden_size: int
|
||||
rms_eps: float
|
||||
hc_pre_eps: float
|
||||
hc_sinkhorn_eps: float
|
||||
hc_post_mult_value: float
|
||||
sinkhorn_repeat: int
|
||||
norm_eps: float
|
||||
n_splits: int
|
||||
hc_mult: int
|
||||
use_pre_mix_in: bool
|
||||
rms_numel: int
|
||||
save_pre_mix: bool = True
|
||||
split_mode: str = "fused"
|
||||
write_aux: bool = False
|
||||
|
||||
kernel: Any = staticmethod(mhc_pre_big_fuse_with_norm_tilelang)
|
||||
|
||||
def dispatch(self, *, n_splits, **fields) -> CompileKey: # type: ignore[override]
|
||||
return self.CompileKey(n_splits=n_splits, **fields)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
max_tokens: int,
|
||||
extra_splits: Iterable[int] = (),
|
||||
**fields,
|
||||
) -> list[CompileKey]:
|
||||
# The split heuristic changes only at 64-token boundaries. Callers that
|
||||
# reach this kernel through another GEMM (the fused post + pre-norm
|
||||
# GEMM) pass the splits that GEMM picks in extra_splits.
|
||||
splits = sorted(
|
||||
set(extra_splits)
|
||||
| (
|
||||
{
|
||||
compute_mhc_pre_num_splits(fields["rms_numel"], num_tokens)
|
||||
for num_tokens in range(1, max_tokens + 1, 64)
|
||||
}
|
||||
if is_deep_gemm_supported()
|
||||
else {1}
|
||||
)
|
||||
)
|
||||
return self._trace_dispatch(self.dispatch)(n_splits=splits, **fields)
|
||||
|
||||
def compile(self, compile_key: CompileKey) -> None:
|
||||
if compile_key not in self._compiled_cache:
|
||||
self._compiled_cache[compile_key] = self.kernel.compile(
|
||||
**asdict(compile_key)
|
||||
)
|
||||
|
||||
def compile_many(self, compile_keys: Iterable[CompileKey]) -> None:
|
||||
missing = list(
|
||||
dict.fromkeys(
|
||||
key for key in compile_keys if key not in self._compiled_cache
|
||||
)
|
||||
)
|
||||
if len(missing) < 2:
|
||||
return super().compile_many(missing)
|
||||
|
||||
# TileLang elaborates serially before compiling independent TIR modules.
|
||||
compiled = self.kernel.par_compile(
|
||||
[asdict(key) for key in missing], num_workers=min(4, len(missing))
|
||||
)
|
||||
self._compiled_cache.update(zip(missing, compiled, strict=True))
|
||||
|
||||
def __call__(self, *tensors, **fields):
|
||||
compile_key = self.dispatch(
|
||||
n_splits=tensors[0].shape[0],
|
||||
**fields,
|
||||
)
|
||||
if not current_platform.is_cuda():
|
||||
return self.kernel(*tensors, **asdict(compile_key))
|
||||
return self._get_or_compile(compile_key)(*tensors)
|
||||
|
||||
|
||||
MHC_PRE_NORM_KERNEL = MHCPreNormKernel()
|
||||
Reference in New Issue
Block a user