[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,602 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import Float32, Int32, Uint32, Uint64
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import recast_val
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
|
||||
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
|
||||
CuTeDSLLaunchSpec,
|
||||
VllmCuTeDSLJitKernel,
|
||||
)
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
def _dcp_merge_topk_and_interleave(vllm_config: Any) -> tuple[int, int]:
|
||||
"""(topk, cp_interleave) the DCP merge runs with.
|
||||
|
||||
Kpool indexers (GLM-5.x) merge pool ids: topk and the interleave are both
|
||||
in units of ``index_kpool`` tokens.
|
||||
"""
|
||||
hf_text_config = vllm_config.model_config.hf_text_config
|
||||
topk = hf_text_config.index_topk
|
||||
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
|
||||
kpool = getattr(hf_text_config, "index_kpool", None) or 1
|
||||
if kpool > 1:
|
||||
topk //= kpool
|
||||
cp_interleave //= kpool
|
||||
return topk, cp_interleave
|
||||
|
||||
|
||||
def stable_topk_from_gathered_candidates_cutedsl(
|
||||
gathered: torch.Tensor,
|
||||
topk: int,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if out is None:
|
||||
out = torch.empty(
|
||||
(gathered.shape[0], topk),
|
||||
dtype=torch.int32,
|
||||
device=gathered.device,
|
||||
)
|
||||
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL(gathered, out, topk=topk)
|
||||
return out
|
||||
|
||||
|
||||
def pack_dcp_topk_candidates_cutedsl(
|
||||
logits: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
packed: torch.Tensor,
|
||||
dcp_rank: int,
|
||||
dcp_world_size: int,
|
||||
cp_interleave: int,
|
||||
row_starts: torch.Tensor | None,
|
||||
) -> None:
|
||||
topk = topk_indices.shape[1]
|
||||
row_starts_arg = row_starts if row_starts is not None else topk_indices
|
||||
_PACK_DCP_TOPK_CANDIDATES_KERNEL(
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
row_starts_arg,
|
||||
logits_stride0=logits.stride(0),
|
||||
logits_stride1=logits.stride(1),
|
||||
topk_stride0=topk_indices.stride(0),
|
||||
topk_stride1=topk_indices.stride(1),
|
||||
packed_stride0=packed.stride(0),
|
||||
packed_stride1=packed.stride(1),
|
||||
packed_stride2=packed.stride(2),
|
||||
num_cols=logits.shape[1],
|
||||
dcp_rank=dcp_rank,
|
||||
dcp_world_size=dcp_world_size,
|
||||
cp_interleave=cp_interleave,
|
||||
has_row_starts=row_starts is not None,
|
||||
topk=topk,
|
||||
block_size=512,
|
||||
)
|
||||
|
||||
|
||||
class PackDCPTopkCandidatesKernel(
|
||||
VllmTritonJitKernel["PackDCPTopkCandidatesKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dcp_rank: int
|
||||
dcp_world_size: int
|
||||
cp_interleave: int
|
||||
has_row_starts: bool
|
||||
topk: int
|
||||
block_size: int
|
||||
|
||||
# These scalars only describe runtime layouts and bounds; the constexpr
|
||||
# fields below own the launch geometry and algorithmic specialization.
|
||||
@staticmethod
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"logits_stride0",
|
||||
"logits_stride1",
|
||||
"topk_stride0",
|
||||
"topk_stride1",
|
||||
"packed_stride0",
|
||||
"packed_stride1",
|
||||
"packed_stride2",
|
||||
"num_cols",
|
||||
]
|
||||
)
|
||||
def kernel(
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
row_starts,
|
||||
logits_stride0,
|
||||
logits_stride1,
|
||||
topk_stride0,
|
||||
topk_stride1,
|
||||
packed_stride0,
|
||||
packed_stride1,
|
||||
packed_stride2,
|
||||
num_cols,
|
||||
DCP_RANK: tl.constexpr,
|
||||
DCP_WORLD_SIZE: tl.constexpr,
|
||||
CP_INTERLEAVE: tl.constexpr,
|
||||
HAS_ROW_STARTS: tl.constexpr,
|
||||
TOPK: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
tile = tl.program_id(1)
|
||||
cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = cols < TOPK
|
||||
|
||||
local_idx = tl.load(
|
||||
topk_indices + row * topk_stride0 + cols * topk_stride1,
|
||||
mask=mask,
|
||||
other=-1,
|
||||
)
|
||||
valid = local_idx >= 0
|
||||
safe_local_idx = tl.maximum(local_idx, 0)
|
||||
|
||||
row_start = 0
|
||||
if HAS_ROW_STARTS:
|
||||
row_start = tl.load(row_starts + row)
|
||||
|
||||
score_col = safe_local_idx + row_start
|
||||
score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0))
|
||||
score = tl.load(
|
||||
logits + row * logits_stride0 + score_col * logits_stride1,
|
||||
mask=mask & valid,
|
||||
other=-float("inf"),
|
||||
)
|
||||
|
||||
global_id = (
|
||||
(safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE)
|
||||
+ DCP_RANK * CP_INTERLEAVE
|
||||
+ safe_local_idx % CP_INTERLEAVE
|
||||
)
|
||||
global_id = tl.where(valid, global_id, -1)
|
||||
|
||||
packed_base = packed + row * packed_stride0 + cols * packed_stride1
|
||||
tl.store(packed_base, score, mask=mask)
|
||||
tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
has_row_starts: bool,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
has_row_starts=has_row_starts,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if dcp_world_size <= 1:
|
||||
return []
|
||||
topk, cp_interleave = _dcp_merge_topk_and_interleave(vllm_config)
|
||||
if topk <= 0:
|
||||
return []
|
||||
|
||||
try:
|
||||
from vllm.distributed.parallel_state import get_dcp_group
|
||||
|
||||
dcp_rank = get_dcp_group().rank_in_group
|
||||
except Exception:
|
||||
dcp_rank = 0
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dcp_rank=dcp_rank,
|
||||
dcp_world_size=dcp_world_size,
|
||||
cp_interleave=cp_interleave,
|
||||
has_row_starts=(False, True),
|
||||
topk=topk,
|
||||
block_size=512,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
fp32_ptr = TritonWarmupTensor(torch.float32)
|
||||
int32_ptr = TritonWarmupTensor(torch.int32)
|
||||
return dict(
|
||||
logits=fp32_ptr,
|
||||
topk_indices=TritonWarmupTensor(torch.int32, shape=(1, compile_key.topk)),
|
||||
packed=fp32_ptr,
|
||||
row_starts_arg=int32_ptr,
|
||||
logits_stride0=1,
|
||||
logits_stride1=1,
|
||||
topk_stride0=1,
|
||||
topk_stride1=1,
|
||||
packed_stride0=1,
|
||||
packed_stride1=1,
|
||||
packed_stride2=1,
|
||||
num_cols=1,
|
||||
dcp_rank=compile_key.dcp_rank,
|
||||
dcp_world_size=compile_key.dcp_world_size,
|
||||
cp_interleave=compile_key.cp_interleave,
|
||||
has_row_starts=compile_key.has_row_starts,
|
||||
topk=compile_key.topk,
|
||||
block_size=compile_key.block_size,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
packed: torch.Tensor,
|
||||
row_starts_arg: torch.Tensor,
|
||||
*,
|
||||
logits_stride0: int,
|
||||
logits_stride1: int,
|
||||
topk_stride0: int,
|
||||
topk_stride1: int,
|
||||
packed_stride0: int,
|
||||
packed_stride1: int,
|
||||
packed_stride2: int,
|
||||
num_cols: int,
|
||||
dcp_rank: int,
|
||||
dcp_world_size: int,
|
||||
cp_interleave: int,
|
||||
has_row_starts: bool,
|
||||
topk: int,
|
||||
block_size: int,
|
||||
) -> LaunchSpec:
|
||||
grid = (topk_indices.shape[0], triton.cdiv(topk, block_size))
|
||||
return grid, dict(
|
||||
row_starts=row_starts_arg,
|
||||
DCP_RANK=dcp_rank,
|
||||
DCP_WORLD_SIZE=dcp_world_size,
|
||||
CP_INTERLEAVE=cp_interleave,
|
||||
HAS_ROW_STARTS=has_row_starts,
|
||||
TOPK=topk,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=8,
|
||||
)
|
||||
|
||||
|
||||
class StableTopKFromGatheredCandidatesKernel(
|
||||
VllmCuTeDSLJitKernel["StableTopKFromGatheredCandidatesKernel.CompileKey"]
|
||||
):
|
||||
tb_size = 512
|
||||
hist_bins = 2048
|
||||
radix_bits = (hist_bins - 1).bit_length()
|
||||
key_bits = Uint64.width
|
||||
radix_passes = (key_bits + radix_bits - 1) // radix_bits
|
||||
final_radix_bits = key_bits - radix_bits * (radix_passes - 1)
|
||||
hist_chunks = (hist_bins + tb_size - 1) // tb_size
|
||||
warps_per_block = tb_size // cute.arch.WARP_SIZE
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
topk: int
|
||||
num_candidates: int
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> Any:
|
||||
tb_size = StableTopKFromGatheredCandidatesKernel.tb_size
|
||||
hist_bins = StableTopKFromGatheredCandidatesKernel.hist_bins
|
||||
radix_bits = StableTopKFromGatheredCandidatesKernel.radix_bits
|
||||
key_bits = StableTopKFromGatheredCandidatesKernel.key_bits
|
||||
radix_passes = StableTopKFromGatheredCandidatesKernel.radix_passes
|
||||
final_radix_bits = StableTopKFromGatheredCandidatesKernel.final_radix_bits
|
||||
hist_chunks = StableTopKFromGatheredCandidatesKernel.hist_chunks
|
||||
warps_per_block = StableTopKFromGatheredCandidatesKernel.warps_per_block
|
||||
topk = compile_key.topk
|
||||
assert hist_bins == 1 << radix_bits
|
||||
assert compile_key.num_candidates % tb_size == 0, (
|
||||
"StableTopKFromGatheredCandidatesKernel requires candidate count "
|
||||
f"to be a multiple of {tb_size}, got {compile_key.num_candidates}"
|
||||
)
|
||||
keys_per_thread = compile_key.num_candidates // tb_size
|
||||
|
||||
@cute.struct
|
||||
class SharedStorage:
|
||||
hist: cute.struct.MemRange[Int32, hist_bins]
|
||||
committed_count: cute.struct.MemRange[Int32, 1]
|
||||
running_count: cute.struct.MemRange[Int32, 1]
|
||||
threshold_bin: cute.struct.MemRange[Int32, 1]
|
||||
threshold_found: cute.struct.MemRange[Int32, 1]
|
||||
include_threshold_bin: cute.struct.MemRange[Int32, 1]
|
||||
prefix_s: cute.struct.Align[cute.struct.MemRange[Uint64, 1], 8]
|
||||
warp_totals: cute.struct.MemRange[Int32, warps_per_block]
|
||||
|
||||
shared_storage = SharedStorage
|
||||
|
||||
@cute.jit
|
||||
def warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32:
|
||||
for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1):
|
||||
offset = 1 << i
|
||||
partial = cute.arch.shuffle_sync_up(
|
||||
val, offset=offset, mask_and_clamp=0
|
||||
)
|
||||
if lane >= offset:
|
||||
val += partial
|
||||
return val
|
||||
|
||||
@cute.jit
|
||||
def block_scan_inclusive_i32(
|
||||
val: Int32,
|
||||
lane: Int32,
|
||||
warp_id: Int32,
|
||||
warp_scratch: cute.Tensor,
|
||||
warps_per_block: int,
|
||||
) -> Int32:
|
||||
prefix = warp_scan_inclusive_i32(val, lane)
|
||||
if lane == Int32(cute.arch.WARP_SIZE - 1):
|
||||
warp_scratch[0, warp_id] = prefix
|
||||
cute.arch.sync_threads()
|
||||
|
||||
if warp_id == Int32(0):
|
||||
warp_total = Int32(0)
|
||||
if lane < Int32(warps_per_block):
|
||||
warp_total = warp_scratch[0, lane]
|
||||
warp_prefix = warp_scan_inclusive_i32(warp_total, lane)
|
||||
if lane < Int32(warps_per_block):
|
||||
warp_scratch[0, lane] = warp_prefix - warp_total
|
||||
cute.arch.sync_threads()
|
||||
|
||||
return prefix + warp_scratch[0, warp_id]
|
||||
|
||||
@cute.jit
|
||||
def stable_key(score: Float32, token_id: Int32) -> Uint64:
|
||||
bits = recast_val(score, Uint32)
|
||||
mask = Uint32(0x80000000)
|
||||
if (bits & Uint32(0x80000000)) != Uint32(0):
|
||||
mask = Uint32(0xFFFFFFFF)
|
||||
score_key = Uint64(bits ^ mask) << Uint64(32)
|
||||
id_key = Uint64(~Uint32(token_id))
|
||||
key = score_key | id_key
|
||||
if token_id < Int32(0):
|
||||
key = Uint64(0)
|
||||
return key
|
||||
|
||||
@cute.jit
|
||||
def prefix_matches(
|
||||
key: Uint64,
|
||||
prefix: Uint64,
|
||||
prefix_bits: Int32,
|
||||
):
|
||||
matches = prefix_bits == Int32(0)
|
||||
if prefix_bits != Int32(0):
|
||||
shift = Int32(key_bits) - prefix_bits
|
||||
matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift))
|
||||
return matches
|
||||
|
||||
@cute.jit
|
||||
def radix_pass(
|
||||
keys: cute.Tensor,
|
||||
output: cute.Tensor,
|
||||
storage,
|
||||
tid: Int32,
|
||||
step: Int32,
|
||||
bits: int,
|
||||
is_final_pass: bool,
|
||||
):
|
||||
hist_smem = storage.hist.get_tensor(cute.make_layout((hist_bins,)))
|
||||
committed_count_smem = storage.committed_count.data_ptr()
|
||||
running_count_smem = storage.running_count.data_ptr()
|
||||
threshold_bin_smem = storage.threshold_bin.data_ptr()
|
||||
threshold_found_smem = storage.threshold_found.data_ptr()
|
||||
include_threshold_bin_smem = storage.include_threshold_bin.data_ptr()
|
||||
prefix_smem = storage.prefix_s.data_ptr()
|
||||
warp_totals_smem = storage.warp_totals.get_tensor(
|
||||
cute.make_layout((1, warps_per_block))
|
||||
)
|
||||
|
||||
prefix_bits = step * Int32(radix_bits)
|
||||
num_bins = 1 << bits
|
||||
block_scan_iterations = (num_bins + tb_size - 1) // tb_size
|
||||
shift = Int32(key_bits) - prefix_bits - Int32(bits)
|
||||
bin_mask = Uint64(num_bins - 1)
|
||||
prefix = prefix_smem.load()
|
||||
|
||||
for chunk in cutlass.range_constexpr(hist_chunks):
|
||||
hist_smem[tid + Int32(chunk * tb_size)] = Int32(0)
|
||||
if tid == Int32(0):
|
||||
running_count_smem.store(committed_count_smem.load())
|
||||
include_threshold_bin_smem.store(Int32(0))
|
||||
threshold_found_smem.store(Int32(0))
|
||||
cute.arch.sync_threads()
|
||||
|
||||
for key_idx in cutlass.range_constexpr(keys_per_thread):
|
||||
key = keys[key_idx]
|
||||
if prefix_matches(key, prefix, prefix_bits):
|
||||
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
|
||||
cute.arch.atomic_add(
|
||||
hist_smem.iterator + bin_idx,
|
||||
Int32(1),
|
||||
sem="relaxed",
|
||||
scope="cta",
|
||||
)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
lane = cute.arch.lane_idx()
|
||||
warp_id = cute.arch.warp_idx()
|
||||
# Each iteration scans one tb_size-wide slice of bins, high to low.
|
||||
iter = Int32(0)
|
||||
threshold_found = threshold_found_smem.load()
|
||||
while threshold_found == Int32(0) and iter < Int32(block_scan_iterations):
|
||||
bin_idx = Int32(num_bins - 1) - (iter * Int32(tb_size) + tid)
|
||||
count = hist_smem[bin_idx]
|
||||
chunk_inclusive = block_scan_inclusive_i32(
|
||||
count,
|
||||
lane,
|
||||
warp_id,
|
||||
warp_totals_smem,
|
||||
warps_per_block,
|
||||
)
|
||||
running_count = running_count_smem.load()
|
||||
prior_in_scan_slice = chunk_inclusive - count
|
||||
remaining = Int32(topk) - running_count - prior_in_scan_slice
|
||||
if count > Int32(0) and remaining > Int32(0) and remaining <= count:
|
||||
threshold_bin_smem.store(bin_idx)
|
||||
if count <= remaining or cutlass.const_expr(is_final_pass):
|
||||
include_threshold_bin_smem.store(Int32(1))
|
||||
threshold_found_smem.store(Int32(1))
|
||||
# Barrier: every thread must finish reading running_count for this
|
||||
# slice before tb_size-1 advances it, else a warp racing ahead to
|
||||
# the store makes a lagging thread double-count the slice total
|
||||
# (-> remaining too small -> threshold too high -> under-fill).
|
||||
cute.arch.sync_threads()
|
||||
if tid == Int32(tb_size - 1):
|
||||
running_count_smem.store(running_count + chunk_inclusive)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
threshold_found = threshold_found_smem.load()
|
||||
iter += Int32(1)
|
||||
|
||||
threshold = threshold_bin_smem.load()
|
||||
should_include_threshold = include_threshold_bin_smem.load() != Int32(0)
|
||||
for key_idx in cutlass.range_constexpr(keys_per_thread):
|
||||
key = keys[key_idx]
|
||||
if prefix_matches(key, prefix, prefix_bits):
|
||||
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
|
||||
selected = bin_idx > threshold
|
||||
if should_include_threshold:
|
||||
selected = selected or bin_idx == threshold
|
||||
if selected:
|
||||
dst = cute.arch.atomic_add(
|
||||
committed_count_smem,
|
||||
Int32(1),
|
||||
sem="relaxed",
|
||||
scope="cta",
|
||||
)
|
||||
if dst < Int32(topk):
|
||||
output[dst] = recast_val(~Uint32(key), Int32)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
pass_finished = include_threshold_bin_smem.load()
|
||||
if tid == Int32(0) and pass_finished == Int32(0):
|
||||
prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift)))
|
||||
cute.arch.sync_threads()
|
||||
return pass_finished
|
||||
|
||||
@cute.kernel
|
||||
def device_kernel(input: cute.Tensor, out: cute.Tensor):
|
||||
row, _, _ = cute.arch.block_idx()
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
input_row = input[row, None, None]
|
||||
output_row = out[row, None]
|
||||
keys = cute.make_rmem_tensor((keys_per_thread,), Uint64)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
storage = smem.allocate(shared_storage, 8)
|
||||
committed_count_smem = storage.committed_count.data_ptr()
|
||||
prefix_smem = storage.prefix_s.data_ptr()
|
||||
for i in range(tid, topk, tb_size):
|
||||
output_row[i] = Int32(-1)
|
||||
|
||||
for key_idx in cutlass.range_constexpr(keys_per_thread):
|
||||
col = tid + Int32(key_idx * tb_size)
|
||||
score = Float32(input_row[col, 0])
|
||||
token_id = Int32(input_row[col, 1])
|
||||
keys[key_idx] = stable_key(score, token_id)
|
||||
|
||||
if tid == Int32(0):
|
||||
committed_count_smem.store(Int32(0))
|
||||
prefix_smem.store(Uint64(0))
|
||||
cute.arch.sync_threads()
|
||||
|
||||
step = Int32(0)
|
||||
finished = Int32(0)
|
||||
while finished == Int32(0) and step < Int32(radix_passes - 1):
|
||||
finished = radix_pass(
|
||||
keys,
|
||||
output_row,
|
||||
storage,
|
||||
tid,
|
||||
step,
|
||||
radix_bits,
|
||||
False,
|
||||
)
|
||||
step += Int32(1)
|
||||
|
||||
if finished == Int32(0):
|
||||
radix_pass(
|
||||
keys,
|
||||
output_row,
|
||||
storage,
|
||||
tid,
|
||||
Int32(radix_passes - 1),
|
||||
final_radix_bits,
|
||||
True,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def host_entrypoint(
|
||||
gathered: cute.Tensor,
|
||||
out: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
grid = (gathered.shape[0], 1, 1)
|
||||
device_kernel(gathered, out).launch(
|
||||
grid=grid,
|
||||
block=(tb_size, 1, 1),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
return host_entrypoint
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(**compile_key_fields)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if dcp_world_size <= 1:
|
||||
return []
|
||||
topk, _ = _dcp_merge_topk_and_interleave(vllm_config)
|
||||
if topk <= 0:
|
||||
return []
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
topk=topk,
|
||||
num_candidates=topk * dcp_world_size,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
|
||||
num_rows = cute.sym_int()
|
||||
gathered = cute.runtime.make_fake_tensor(
|
||||
Float32,
|
||||
(num_rows, compile_key.num_candidates, 2),
|
||||
stride=(cute.sym_int64(divisibility=2), 2, 1),
|
||||
assumed_align=8,
|
||||
)
|
||||
out = make_fake_tensor(
|
||||
Int32,
|
||||
(num_rows, compile_key.topk),
|
||||
divisibility=1,
|
||||
)
|
||||
return gathered, out
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
gathered: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
*,
|
||||
topk: int,
|
||||
) -> CuTeDSLLaunchSpec[CompileKey]:
|
||||
compile_key = self.dispatch(
|
||||
topk=topk,
|
||||
num_candidates=gathered.shape[1],
|
||||
)
|
||||
return compile_key, (gathered, out)
|
||||
|
||||
|
||||
_PACK_DCP_TOPK_CANDIDATES_KERNEL = PackDCPTopkCandidatesKernel()
|
||||
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL = StableTopKFromGatheredCandidatesKernel()
|
||||
@@ -59,10 +59,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
|
||||
TritonW4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
|
||||
XPUW4A8IntLinearKernel,
|
||||
XPUwNa16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
|
||||
ZentorchWNA16LinearKernel,
|
||||
)
|
||||
@@ -91,9 +87,6 @@ from vllm.model_executor.kernels.linear.mxfp4.marlin import (
|
||||
from vllm.model_executor.kernels.linear.mxfp4.pytorch import (
|
||||
TorchMxFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp4.xpu import (
|
||||
XPUMxFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp6 import (
|
||||
MxFp6LinearKernel,
|
||||
MxFp6LinearLayerConfig,
|
||||
@@ -132,9 +125,6 @@ from vllm.model_executor.kernels.linear.mxfp8.pytorch import (
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.xpu import (
|
||||
XPUMxFp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.nvfp4 import (
|
||||
NvFp4LinearKernel,
|
||||
NvFp4LinearLayerConfig,
|
||||
@@ -228,11 +218,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
XPUFp8BlockScaledMMKernel,
|
||||
XPUW8A8FP8LinearKernel,
|
||||
XPUW8A16FP8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
|
||||
ZentorchInt8ScaledMMLinearKernel,
|
||||
)
|
||||
@@ -373,13 +358,6 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
EmulationMxfp6LinearKernel,
|
||||
EmulationMxfp4LinearKernel,
|
||||
},
|
||||
"xpu": {
|
||||
XPUW8A8FP8LinearKernel,
|
||||
XPUFp8BlockScaledMMKernel,
|
||||
},
|
||||
"xpu_woq": {
|
||||
XPUW8A16FP8LinearKernel,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -444,7 +422,6 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]
|
||||
HummingInt8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [AiterInt8ScaledMMLinearKernel, TritonInt8ScaledMMLinearKernel],
|
||||
PlatformEnum.XPU: [TritonInt8ScaledMMLinearKernel],
|
||||
}
|
||||
|
||||
# in priority/performance order (when available)
|
||||
@@ -473,12 +450,6 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUW8A16FP8LinearKernel,
|
||||
XPUW8A8FP8LinearKernel,
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@@ -504,11 +475,6 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
|
||||
CPUFp8W8A8BlockScaledMMKernel, # W8A8 preferred; falls back to W8A16 below
|
||||
CPUFp8BlockScaledMMKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUFp8BlockScaledMMKernel,
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
BlockWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
_POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
|
||||
@@ -522,9 +488,6 @@ _POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]
|
||||
PlatformEnum.CPU: [
|
||||
# To be added
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUW8A16FP8LinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
# in priority/performance order (when available)
|
||||
@@ -545,10 +508,6 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
|
||||
ConchLinearKernel,
|
||||
ExllamaLinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUW4A8IntLinearKernel,
|
||||
XPUwNa16LinearKernel,
|
||||
],
|
||||
PlatformEnum.CPU: [
|
||||
Dynamic4bitLinearKernel,
|
||||
ZentorchWNA16LinearKernel,
|
||||
@@ -574,11 +533,6 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
EmulationMxfp8LinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUMxFp8LinearKernel,
|
||||
TorchMxFp8LinearKernel,
|
||||
EmulationMxfp8LinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
_POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
|
||||
@@ -626,10 +580,6 @@ _POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = {
|
||||
AiterMxfp4LinearKernel,
|
||||
EmulationMxfp4LinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUMxFp4LinearKernel,
|
||||
TorchMxFp4LinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
# TODO make all kernels inherit from MMLinearKernel
|
||||
@@ -1348,7 +1298,6 @@ __all__ = [
|
||||
"MacheteLinearKernel",
|
||||
"MarlinLinearKernel",
|
||||
"TritonW4A16LinearKernel",
|
||||
"XPUW4A8IntLinearKernel",
|
||||
"XPUwNa16LinearKernel",
|
||||
"init_mxfp8_linear_kernel",
|
||||
"Mxfp8LinearKernel",
|
||||
@@ -1373,7 +1322,6 @@ __all__ = [
|
||||
"FlashInferCutlassMxfp8LinearKernel",
|
||||
"FlashInferTrtllmMxfp8LinearKernel",
|
||||
"MarlinMxfp8LinearKernel",
|
||||
"XPUMxFp8LinearKernel",
|
||||
"TorchMxFp8LinearKernel",
|
||||
"EmulationMxfp8LinearKernel",
|
||||
"CutlassNvFp4LinearKernel",
|
||||
|
||||
@@ -38,10 +38,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
|
||||
TritonW4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
|
||||
XPUW4A8IntLinearKernel,
|
||||
XPUwNa16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
|
||||
ZentorchWNA16LinearKernel,
|
||||
)
|
||||
@@ -60,7 +56,6 @@ __all__ = [
|
||||
"RDNA3W4A16LinearKernel",
|
||||
"RDNAHybridW4A16LinearKernel",
|
||||
"TritonW4A16LinearKernel",
|
||||
"XPUW4A8IntLinearKernel",
|
||||
"XPUwNa16LinearKernel",
|
||||
"ZentorchWNA16LinearKernel",
|
||||
]
|
||||
|
||||
@@ -1,218 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
|
||||
|
||||
_XPUWNA16_SUPPORTED_QUANT_TYPES = (scalar_types.uint4, scalar_types.uint4b8)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class XPUwNa16LinearKernel(MPLinearKernel):
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return -1
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUwNa16 only supported on XPU"
|
||||
|
||||
if c.act_type != torch.bfloat16 and c.act_type != torch.float16:
|
||||
return False, "XPUwNa16 only supports BF16/FP16 activations"
|
||||
|
||||
if c.weight_type not in _XPUWNA16_SUPPORTED_QUANT_TYPES:
|
||||
return (
|
||||
False,
|
||||
f"Quant type ({c.weight_type}) not supported by "
|
||||
"XPUwNa16, supported types are: "
|
||||
f"{_XPUWNA16_SUPPORTED_QUANT_TYPES}",
|
||||
)
|
||||
if c.group_size != -1 and c.group_size % 32 != 0:
|
||||
return (
|
||||
False,
|
||||
f"Group size ({c.group_size}) not supported by "
|
||||
"XPUwNa16, supported group sizes are multiples of 32",
|
||||
)
|
||||
|
||||
if c.partition_weight_shape[0] % 32 != 0:
|
||||
return (
|
||||
False,
|
||||
f"Input size ({c.partition_weight_shape[0]}) not supported by "
|
||||
"XPUwNa16, supported sizes are multiples of 32",
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
# Default name since marlin requires empty parameter for zp,
|
||||
# TODO: remove this requirement from marlin (allow optional tensors)
|
||||
if self.w_zp_name is None:
|
||||
self.w_zp_name = "w_zp"
|
||||
|
||||
need_transpose = False
|
||||
qweight_shape = getattr(layer, self.w_q_name).shape
|
||||
scale_shape = getattr(layer, self.w_s_name).shape
|
||||
# gptq marlin and compressed tensors wna16 expect different default
|
||||
# layouts for weight and scale, so we check the shapes to determine
|
||||
# if we need to transpose
|
||||
if qweight_shape[0] != scale_shape[0]:
|
||||
need_transpose = True
|
||||
|
||||
if need_transpose:
|
||||
getattr(layer, self.w_q_name).data = (
|
||||
getattr(layer, self.w_q_name).data.t().contiguous()
|
||||
)
|
||||
getattr(layer, self.w_s_name).data = getattr(layer, self.w_s_name).data
|
||||
else:
|
||||
getattr(layer, self.w_s_name).data = (
|
||||
getattr(layer, self.w_s_name).data.t().contiguous()
|
||||
)
|
||||
|
||||
if self.config.zero_points:
|
||||
# (FIXME): maybe zero points should also be transposed.
|
||||
getattr(layer, self.w_zp_name).data = (
|
||||
getattr(layer, self.w_zp_name).data.t().contiguous()
|
||||
)
|
||||
else:
|
||||
weight_zero_point = torch.Tensor([8]).to(torch.int8).to("xpu")
|
||||
setattr(
|
||||
layer, self.w_zp_name, Parameter(weight_zero_point, requires_grad=False)
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a16(
|
||||
reshaped_x,
|
||||
w_q.t(),
|
||||
bias if bias is not None else None,
|
||||
w_s,
|
||||
w_zp,
|
||||
self.config.group_size,
|
||||
None, # Retained by the external XPU op ABI.
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
class XPUW4A8IntLinearKernel(MPLinearKernel):
|
||||
"""XPU kernel for W4A8 integer quantization using oneDNN int4_gemm_w4a8.
|
||||
|
||||
Weights are symmetric group-quantized int4 packed as uint4.
|
||||
Activations are dynamically quantized per-token to symmetric int8.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return -1
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUW4A8Int only supported on XPU"
|
||||
if c.act_type not in (torch.bfloat16, torch.float16):
|
||||
return False, "XPUW4A8Int requires BF16/FP16 activations"
|
||||
if c.weight_type != scalar_types.int4:
|
||||
return (
|
||||
False,
|
||||
f"XPUW4A8Int requires int4 weights, got {c.weight_type}",
|
||||
)
|
||||
if c.zero_points:
|
||||
return False, "XPUW4A8Int only supports symmetric weight quantization"
|
||||
if c.group_size != -1 and c.group_size % 32 != 0:
|
||||
return (
|
||||
False,
|
||||
f"Group size ({c.group_size}) not supported by XPUW4A8Int, "
|
||||
"must be a multiple of 32",
|
||||
)
|
||||
in_size, out_size = c.partition_weight_shape
|
||||
if in_size % 8 != 0 or out_size % 8 != 0:
|
||||
return (
|
||||
False,
|
||||
f"in/out sizes ({in_size}, {out_size}) must be multiples of 8",
|
||||
)
|
||||
|
||||
if c.act_type != torch.float16:
|
||||
logger.warning_once(
|
||||
"XPUW4A8IntLinearKernel is running with model dtype %s, "
|
||||
"but int4_gemm_w4a8 produces float16 output. Recommend "
|
||||
"setting --dtype float16 for best performance.",
|
||||
c.act_type,
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
def _pack_int4_weight(self, w: torch.Tensor) -> torch.Tensor:
|
||||
# w is [N, K] int8 with values in [-8, 7]
|
||||
w_u4 = w.to(torch.int32) + 8 # shift to [0, 15]
|
||||
w_u4 = w_u4.reshape(w.shape[0], w.shape[1] // 8, 8) # [N, K/8, 8]
|
||||
shifts = torch.arange(0, 32, 4, dtype=torch.int32, device=w.device)
|
||||
packed = ((w_u4 & 0xF) << shifts[None, None, :]).sum(dim=2).to(torch.int32)
|
||||
return packed
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight_scale.data = layer.weight_scale.data.t().contiguous()
|
||||
|
||||
device = layer.weight_packed.device
|
||||
# TODO: support asymmetric quantization
|
||||
weight_zero_point = torch.tensor([8], dtype=torch.int8, device=device)
|
||||
layer.weight_zero_point = Parameter(weight_zero_point, requires_grad=False)
|
||||
|
||||
# weight_packed is [out, in] int8, signed int4 values in [-8, 7]
|
||||
w = layer.weight_packed.data # [out, in]
|
||||
|
||||
# TODO: implement asym case
|
||||
packed = self._pack_int4_weight(w) # [out, in/8] packed uint4
|
||||
|
||||
replace_parameter(
|
||||
layer,
|
||||
self.w_q_name,
|
||||
torch.nn.Parameter(packed, requires_grad=False),
|
||||
)
|
||||
|
||||
# Free the original unpacked int8 weight (still registered as "weight")
|
||||
# to avoid double-storing both int8 [N, K] and int32 [N, K/8] in memory.
|
||||
delattr(layer, "weight")
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = x.reshape(-1, x.shape[-1]) # [M, K]
|
||||
from vllm._xpu_ops import xpu_ops as ops
|
||||
|
||||
# TODO: static and asymmetric quantization case
|
||||
# Common code for CompressedTensorsW4A8Int does not read act symmetry data
|
||||
quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref(
|
||||
reshaped_x, True, 8
|
||||
)
|
||||
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a8(
|
||||
quant_x,
|
||||
x_scale,
|
||||
x_zero,
|
||||
layer.weight_packed.t(),
|
||||
layer.weight_scale,
|
||||
layer.weight_zero_point,
|
||||
self.config.group_size,
|
||||
None, # Retained by the external XPU op ABI.
|
||||
bias,
|
||||
)
|
||||
|
||||
return out.to(x.dtype)
|
||||
@@ -1,58 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
xpu_mxfp4_quantize as quant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
|
||||
|
||||
class XPUMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
"""MXFP4 W4A4 GEMM on XPU."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUMxFp4 only support on XPU"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if config.activation_quant_key != kMxfp4Dynamic:
|
||||
return False, "only supports MXFP4 dynamic activation"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
weight = layer.weight.view(torch.float4_e2m1fn_x2)
|
||||
replace_parameter(layer, "weight", weight.data.t())
|
||||
|
||||
weight_scale = layer.weight_scale.view(torch.float8_e8m0fnu)
|
||||
weight_scale = weight_scale.t().contiguous()
|
||||
replace_parameter(layer, "weight_scale", weight_scale.data)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
out_dtype = x.dtype
|
||||
x_fp4, x_blockscale = quant_mxfp4(x)
|
||||
return torch.ops._xpu_C.fp4_gemm(
|
||||
x_fp4,
|
||||
layer.weight,
|
||||
x_blockscale,
|
||||
layer.weight_scale,
|
||||
out_dtype,
|
||||
bias,
|
||||
)
|
||||
@@ -1,82 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
xpu_mxfp8_quantize as quant_mxfp8,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
|
||||
|
||||
class XPUMxFp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""MXFP8 W8A8 GEMM on XPU."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUMxFp8 only support on XPU"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Checkpoint scale is [N, K//32] (one E8M0 scale per 32 elements).
|
||||
# oneDNN fp8_gemm requires contiguous [K//32, N] layout. Store the
|
||||
# transposed contiguous buffer as a .t() view so that:
|
||||
# - dequantization consumers still see the checkpoint shape
|
||||
# - apply_weights recovers the oneDNN layout via .t() at zero cost
|
||||
weight_scale = layer.weight_scale.view(torch.float8_e8m0fnu)
|
||||
scale_kn = weight_scale.data.t().contiguous()
|
||||
replace_parameter(layer, "weight_scale", scale_kn.t())
|
||||
|
||||
if getattr(layer, "is_bmm", False):
|
||||
self._prepare_bmm_params(layer, scale_kn)
|
||||
|
||||
def _prepare_bmm_params(
|
||||
self, layer: torch.nn.Module, scale_kn: torch.Tensor
|
||||
) -> None:
|
||||
"""Precompute batched weight and scale for grouped fp8_bmm (e.g. wo_a).
|
||||
|
||||
Splits scale [K//32, N_total] into [G, K//32, N_per_group] and weight
|
||||
[N_total, K] into contiguous [G, K, N_per_group] for batch GEMM.
|
||||
"""
|
||||
batch = layer.bmm_batch_size
|
||||
k_blocks, n_blocks = scale_kn.shape
|
||||
layer.bmm_scale = (
|
||||
scale_kn.reshape(k_blocks, batch, n_blocks // batch)
|
||||
.permute(1, 0, 2)
|
||||
.contiguous()
|
||||
)
|
||||
w = layer.weight
|
||||
n_total, k = w.shape
|
||||
layer.bmm_weight = (
|
||||
w.reshape(batch, n_total // batch, k).permute(0, 2, 1).contiguous()
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
out_dtype = x.dtype
|
||||
x_fp8, x_scale = quant_mxfp8(x)
|
||||
# Weight is [N, K]; .t() gives a [K, N] view without copying.
|
||||
# Scale is stored as [N, K//32]; .t() recovers the contiguous
|
||||
# [K//32, N] buffer that oneDNN expects.
|
||||
return torch.ops._xpu_C.fp8_gemm(
|
||||
x_fp8,
|
||||
layer.weight.t(),
|
||||
out_dtype,
|
||||
x_scale,
|
||||
layer.weight_scale.t(),
|
||||
bias,
|
||||
)
|
||||
@@ -42,9 +42,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
XPUFp8BlockScaledMMKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
|
||||
ZentorchInt8ScaledMMLinearKernel,
|
||||
)
|
||||
@@ -73,5 +70,4 @@ __all__ = [
|
||||
"CPUFp8BlockScaledMMKernel",
|
||||
"CPUFp8PerTensorScaledMMLinearKernel",
|
||||
"CPUFP8W8A8ScaledMMLinearKernel",
|
||||
"XPUFp8BlockScaledMMKernel",
|
||||
]
|
||||
|
||||
@@ -1,284 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kFp8StaticTokenSym,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel
|
||||
from .ScaledMMLinearKernel import FP8ScaledMMLinearKernel, FP8ScaledMMLinearLayerConfig
|
||||
|
||||
|
||||
class XPUW8A8FP8LinearKernel(FP8ScaledMMLinearKernel):
|
||||
_SUPPORTED_ACT_QUANT_KEYS = {
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticTensorSym,
|
||||
kFp8StaticTokenSym,
|
||||
}
|
||||
_SUPPORTED_WEIGHT_QUANT_KEYS = {
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUW8A8FP8Linear only support on XPU"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if c.weight_quant_key not in cls._SUPPORTED_WEIGHT_QUANT_KEYS:
|
||||
return (
|
||||
False,
|
||||
"XPUW8A8FP8Linear only support per-channel and per-tensor quantization",
|
||||
)
|
||||
if c.activation_quant_key not in cls._SUPPORTED_ACT_QUANT_KEYS:
|
||||
return (
|
||||
False,
|
||||
"XPUW8A8FP8Linear only support per-tensor and per-token activation "
|
||||
"quantization",
|
||||
)
|
||||
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
|
||||
return False, "XPUW8A8FP8Linear only support FP8 weight dtype"
|
||||
if c.activation_quant_key.dtype not in {
|
||||
torch.float8_e5m2,
|
||||
torch.float8_e4m3fn,
|
||||
}:
|
||||
return False, "XPUW8A8FP8Linear only support FP8 activation dtype"
|
||||
return True, None
|
||||
|
||||
def __init__(
|
||||
self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str]
|
||||
) -> None:
|
||||
super().__init__(c, layer_param_names)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""Ensure weight is stored as C-contiguous [K, N] (KN layout).
|
||||
|
||||
Checkpoints store weight as [N, K]; fp8_gemm requires [K, N],
|
||||
C-contiguous. Three incoming layouts are possible:
|
||||
• [N, K] C-contiguous ← direct checkpoint → .t().contiguous()
|
||||
• [K, N] Fortran-order ← fp8.py's weight.t() → .contiguous()
|
||||
• [K, N] C-contiguous ← already correct → no-op
|
||||
|
||||
For square weights (K == N) the shape is ambiguous; contiguity is used
|
||||
as a proxy: C-contiguous ≡ checkpoint [N, K] (needs transpose);
|
||||
Fortran-order ≡ fp8.py already transposed (needs only contiguous).
|
||||
"""
|
||||
K = getattr(layer, "input_size_per_partition", self.config.weight_shape[1])
|
||||
N = getattr(layer, "output_size_per_partition", self.config.weight_shape[0])
|
||||
w = layer.weight
|
||||
|
||||
if w.shape not in {(K, N), (N, K)}:
|
||||
raise ValueError(
|
||||
f"XPUFP8ScaledMM expects weight shape ({K},{N}) or ({N},{K}), "
|
||||
f"but got {tuple(w.shape)}"
|
||||
)
|
||||
|
||||
needs_transpose = w.shape == (N, K) if K != N else w.is_contiguous()
|
||||
layer_weight = w.t() if needs_transpose else w
|
||||
replace_parameter(layer, "weight", layer_weight)
|
||||
ws = layer.weight_scale
|
||||
if ws.numel() == 1:
|
||||
replace_parameter(layer, "weight_scale", ws.reshape(1))
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
# B is C-contiguous [K, N] from process_weights_after_loading.
|
||||
# fp8_gemm routes on scale dtype (float32) and numel:
|
||||
# As [1] → per-tensor (numel==1 branch)
|
||||
# As [M,1] → per-token (group={1,K} branch, broadcast across K)
|
||||
# Bs [1] → per-tensor
|
||||
# Bs [N] → per-channel (mask=bit1 branch)
|
||||
# No shape manipulation needed here.
|
||||
output = torch.ops._xpu_C.fp8_gemm(A, B, out_dtype, As, Bs, bias)
|
||||
return output.view(*output_shape)
|
||||
|
||||
|
||||
class XPUW8A16FP8LinearKernel(FP8ScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUW8A16FP8Linear only support on XPU"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if c.weight_quant_key not in {kFp8StaticChannelSym, kFp8StaticTensorSym}:
|
||||
return (
|
||||
False,
|
||||
"XPUW8A16FP8Linear only support per-channel and per-tensor "
|
||||
"quantization",
|
||||
)
|
||||
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
|
||||
return False, "XPUW8A16FP8Linear only support FP8 weight dtype"
|
||||
return True, None
|
||||
|
||||
def __init__(
|
||||
self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str]
|
||||
) -> None:
|
||||
assert self.can_implement(c)[0]
|
||||
assert self.is_supported()[0]
|
||||
self.config = c
|
||||
self.layer_param_names = layer_param_names
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# fp8_gemm_w8a16 expects weight in [in, out] layout.
|
||||
# Transpose if weight is still in [out, in] layout.
|
||||
# For square matrices, use contiguity as tie-breaker:
|
||||
# checkpoint weights are contiguous, .t() views are not.
|
||||
weight = layer.weight
|
||||
out_features, in_features = self.config.weight_shape
|
||||
|
||||
if weight.shape == (out_features, in_features) and (
|
||||
in_features != out_features or weight.is_contiguous()
|
||||
):
|
||||
replace_parameter(layer, "weight", weight.data.t())
|
||||
# else: already in [in, out] layout — no-op
|
||||
|
||||
weight_scale = layer.weight_scale.t().contiguous()
|
||||
replace_parameter(layer, "weight_scale", weight_scale.data)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
weight = layer.weight
|
||||
weight_scale = layer.weight_scale
|
||||
return torch.ops._xpu_C.fp8_gemm_w8a16(x, weight, weight_scale, bias)
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
|
||||
|
||||
class XPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUFp8BlockScaledMM only support on XPU"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
super().process_weights_after_loading(layer)
|
||||
scale_attr = (
|
||||
"weight_scale_inv" if hasattr(layer, "weight_scale_inv") else "weight_scale"
|
||||
)
|
||||
scale = getattr(layer, scale_attr)
|
||||
|
||||
# Ragged N (N % block_n != 0): oneDNN needs n_blocks to divide N.
|
||||
# Weight untouched; only repeat scale rows to a finer N-group gn that
|
||||
# divides both N and block_n (gn = gcd(N, block_n)):
|
||||
# scale [ceil(N/block_n), K/block_k] --> [N/gn, K/block_k]
|
||||
# oneDNN only accepts gn that is a multiple of 16, and gcd(N, block_n)
|
||||
# is a power of two (block_n=128), so gn must be >= 16. No-op when
|
||||
# N % block_n == 0.
|
||||
block_n, block_k = self.weight_group_shape
|
||||
N, K = layer.weight.shape
|
||||
if N % block_n != 0:
|
||||
gn = math.gcd(N, block_n)
|
||||
assert gn % 16 == 0, (
|
||||
f"XPU block-scaled FP8: N ({N}) yields group width {gn}, but "
|
||||
f"oneDNN only supports multiples of 16; this weight shape is "
|
||||
f"unsupported."
|
||||
)
|
||||
col_start = torch.arange(N // gn, device=scale.device) * gn
|
||||
src_idx = torch.div(col_start, block_n, rounding_mode="floor")
|
||||
scale = scale.index_select(0, src_idx).contiguous()
|
||||
|
||||
# Ragged K needs the runtime activation scale expanded too, which we
|
||||
# don't handle; DeepSeek/GLM keep K block-aligned, so fail loudly.
|
||||
assert K % block_k == 0, (
|
||||
f"XPU block-scaled FP8 requires K ({K}) to be a multiple of the "
|
||||
f"weight block size ({block_k}); ragged-K weights are unsupported."
|
||||
)
|
||||
|
||||
# Checkpoint scale is [n_blocks, k_blocks] (one value per block tile).
|
||||
# oneDNN fp8_gemm requires contiguous [k_blocks, n_blocks] layout.
|
||||
# We store the transposed contiguous buffer as a .t() view so that:
|
||||
# - MLA's scaled_dequantize still sees [n_blocks, k_blocks] shape
|
||||
# - apply_block_scaled_mm recovers the contiguous buffer via .t()
|
||||
scale_kn = scale.data.t().contiguous() # [k_blocks, n_blocks]
|
||||
replace_parameter(layer, scale_attr, scale_kn.t()) # view: [n_blocks, k_blocks]
|
||||
|
||||
if getattr(layer, "is_bmm", False):
|
||||
self._prepare_bmm_params(layer, scale_kn)
|
||||
|
||||
def _prepare_bmm_params(
|
||||
self, layer: torch.nn.Module, scale_kn: torch.Tensor
|
||||
) -> None:
|
||||
"""Precompute batched weight and scale for grouped fp8_bmm (e.g. wo_a).
|
||||
|
||||
Splits scale [k_blocks, n_blocks] into [G, k_blocks, n_blocks_per_group]
|
||||
and weight [N_total, K] into [G, K, N_per_group] for batch GEMM.
|
||||
"""
|
||||
batch = layer.bmm_batch_size
|
||||
k_blocks, n_blocks = scale_kn.shape
|
||||
layer.bmm_scale = (
|
||||
scale_kn.reshape(k_blocks, batch, n_blocks // batch)
|
||||
.permute(1, 0, 2)
|
||||
.contiguous()
|
||||
)
|
||||
w = layer.weight
|
||||
N_total, K = w.shape
|
||||
layer.bmm_weight = w.reshape(batch, N_total // batch, K).permute(
|
||||
0, 2, 1
|
||||
) # [G, K, N_per_group]
|
||||
|
||||
def apply_block_scaled_mm(
|
||||
self,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# B is [N, K]; .t() gives [K, N] view (no copy).
|
||||
# Bs is stored as [n_blocks, k_blocks] view; .t() recovers the
|
||||
# contiguous [k_blocks, n_blocks] buffer that oneDNN expects.
|
||||
return torch.ops._xpu_C.fp8_gemm(
|
||||
A,
|
||||
B.t(),
|
||||
self.config.out_dtype,
|
||||
As,
|
||||
Bs.t(),
|
||||
torch.Tensor(),
|
||||
)
|
||||
@@ -1,36 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .aiter import *
|
||||
from .cpu import *
|
||||
from .tilelang import *
|
||||
from .torch import *
|
||||
from .triton import *
|
||||
|
||||
__all__ = [
|
||||
"mhc_pre_cuda",
|
||||
"mhc_post_cuda",
|
||||
"mhc_fused_post_pre_cuda",
|
||||
"hc_head_fused_kernel_cuda",
|
||||
"mhc_pre_aiter",
|
||||
"mhc_pre_delayed_aiter",
|
||||
"mhc_fused_post_pre_delayed_rms_norm_aiter",
|
||||
"mhc_post_aiter",
|
||||
"mhc_fused_post_pre_aiter",
|
||||
"hc_head_fused_aiter",
|
||||
"mhc_pre_cpu",
|
||||
"mhc_post_cpu",
|
||||
"hc_head_fused_cpu",
|
||||
"mhc_pre_tilelang",
|
||||
"mhc_post_tilelang",
|
||||
"mhc_fused_post_pre_tilelang",
|
||||
"hc_head_fused_tilelang",
|
||||
"mhc_pre_torch",
|
||||
"mhc_post_torch",
|
||||
"mhc_fused_post_pre_torch",
|
||||
"hc_head_fused_torch",
|
||||
"mhc_pre_mix_triton",
|
||||
"mhc_pre_triton",
|
||||
"mhc_post_triton",
|
||||
"mhc_fused_post_pre_triton",
|
||||
"hc_head_fused_triton",
|
||||
]
|
||||
@@ -1,428 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
def mhc_pre_aiter(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
|
||||
hc_scale: shape (3,), dtype torch.float32
|
||||
hc_base: shape (hc_mult3,), dtype torch.float32
|
||||
rms_eps: RMS normalization epsilon
|
||||
hc_pre_eps: pre-mix epsilon
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
n_splits: split-k factor;
|
||||
norm_weight: optional RMSNorm weight fused into the pre kernel
|
||||
norm_eps: epsilon for the fused RMSNorm when norm_weight is set
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
|
||||
"""
|
||||
hidden_size = residual.shape[-1]
|
||||
assert hidden_size % 256 == 0
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_pre(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_pre_aiter_fake(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
# Create empty tensors with correct shapes for meta device / shape inference
|
||||
post_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
1,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input = torch.empty(
|
||||
*outer_shape,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
return post_mix, comb_mix, layer_input
|
||||
|
||||
|
||||
def mhc_pre_delayed_aiter(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
sublayer_out: torch.Tensor | None = None,
|
||||
post_layer_mix: torch.Tensor | None = None,
|
||||
comb_res_mix: torch.Tensor | None = None,
|
||||
residual_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""MHC pre with the pre-mix carried in from the previous sublayer.
|
||||
|
||||
Matches ``mhc_pre_delayed_torch``: the stream collapse uses *pre_mix*
|
||||
rather than the gate computed here, and that gate is returned as the
|
||||
pre-mix for the next sublayer seam.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
|
||||
hc_scale: shape (3,), dtype torch.float32
|
||||
hc_base: shape (hc_mult3,), dtype torch.float32
|
||||
rms_eps: RMS normalization epsilon
|
||||
hc_pre_eps: pre-mix epsilon
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
pre_mix: shape (..., hc_mult) from the previous sublayer, or None at
|
||||
model entry to select residual stream zero.
|
||||
sublayer_out: attention or FFN output, shape (..., hidden_size). When
|
||||
given with the two mixes below, the preceding post block is
|
||||
applied here so AITER can fold it into the pre projection.
|
||||
post_layer_mix: shape (..., hc_mult, 1), post gate for that block.
|
||||
comb_res_mix: shape (..., hc_mult, hc_mult), residual comb for it.
|
||||
residual_out: shape (..., hc_mult, hidden_size), written with the post
|
||||
block's new residual. Required exactly when the post is folded in.
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult, 1), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
next_pre_mix: shape (..., hc_mult), dtype torch.float32
|
||||
|
||||
"""
|
||||
hidden_size = residual.shape[-1]
|
||||
assert hidden_size % 256 == 0
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_pre_delayed(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
sublayer_out,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
residual_out,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_pre_delayed_aiter_fake(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
sublayer_out: torch.Tensor | None = None,
|
||||
post_layer_mix: torch.Tensor | None = None,
|
||||
comb_res_mix: torch.Tensor | None = None,
|
||||
residual_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
return (
|
||||
torch.empty(
|
||||
*outer_shape, hc_mult, 1, dtype=torch.float32, device=residual.device
|
||||
),
|
||||
torch.empty(
|
||||
*outer_shape, hc_mult, hc_mult, dtype=torch.float32, device=residual.device
|
||||
),
|
||||
torch.empty(
|
||||
*outer_shape, hidden_size, dtype=torch.bfloat16, device=residual.device
|
||||
),
|
||||
torch.empty(*outer_shape, hc_mult, dtype=torch.float32, device=residual.device),
|
||||
)
|
||||
|
||||
|
||||
def mhc_fused_post_pre_delayed_rms_norm_aiter(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None,
|
||||
sublayer_out: torch.Tensor | None,
|
||||
post_layer_mix: torch.Tensor | None,
|
||||
comb_res_mix: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
norm_eps: float,
|
||||
residual_out: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
""":func:`mhc_pre_delayed_aiter` with the RMSNorm of the collapse folded in:
|
||||
``layer_input`` comes back normalised with ``norm_weight`` / ``norm_eps``."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_fused_post_pre_delayed_rms_norm(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
sublayer_out,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
residual_out,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_fused_post_pre_delayed_rms_norm_aiter_fake(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None,
|
||||
sublayer_out: torch.Tensor | None,
|
||||
post_layer_mix: torch.Tensor | None,
|
||||
comb_res_mix: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
norm_eps: float,
|
||||
residual_out: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return _mhc_pre_delayed_aiter_fake(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
|
||||
def mhc_post_aiter(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
hidden_size = residual.shape[-1]
|
||||
|
||||
assert hidden_size % 256 == 0
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_post(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_post_aiter_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(residual)
|
||||
|
||||
|
||||
def mhc_fused_post_pre_aiter(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
tile_n: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Fused mHC post + next mHC pre on ROCm via AITER.
|
||||
|
||||
Returns residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur.
|
||||
"""
|
||||
hidden_size = residual.shape[-1]
|
||||
assert hidden_size % 256 == 0
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_fused_post_pre(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_fused_post_pre_aiter_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
tile_n: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
post_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
1,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
comb_mix = torch.empty(
|
||||
*outer_shape,
|
||||
hc_mult,
|
||||
hc_mult,
|
||||
dtype=torch.float32,
|
||||
device=residual.device,
|
||||
)
|
||||
layer_input = torch.empty(
|
||||
*outer_shape,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
return torch.empty_like(residual), post_mix, comb_mix, layer_input
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre_aiter",
|
||||
op_func=mhc_pre_aiter,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_pre_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre_delayed_aiter",
|
||||
op_func=mhc_pre_delayed_aiter,
|
||||
mutates_args=["residual_out"],
|
||||
fake_impl=_mhc_pre_delayed_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_fused_post_pre_delayed_rms_norm_aiter",
|
||||
op_func=mhc_fused_post_pre_delayed_rms_norm_aiter,
|
||||
mutates_args=["residual_out"],
|
||||
fake_impl=_mhc_fused_post_pre_delayed_rms_norm_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_post_aiter",
|
||||
op_func=mhc_post_aiter,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_post_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_fused_post_pre_aiter",
|
||||
op_func=mhc_fused_post_pre_aiter,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_fused_post_pre_aiter_fake,
|
||||
)
|
||||
@@ -1,96 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
|
||||
def mhc_pre_cpu(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""CPU-ported mHC pre block (see `mhc_pre_torch` for the eager reference).
|
||||
|
||||
The ported kernel (`hc_pre_fused_cpu`) only exposes one merged `hc_eps`
|
||||
(used for both `hc_pre_eps`/`hc_sinkhorn_eps`) and hardcodes the post-mix
|
||||
multiplier to 2.0 -- true of every real call site in
|
||||
`models/deepseek_v4/cpu/model.py`, so this is not a capability loss here.
|
||||
"""
|
||||
assert n_splits == 1, "mhc_pre_cpu does not support n_splits != 1"
|
||||
assert hc_pre_eps == hc_sinkhorn_eps, (
|
||||
"mhc_pre_cpu requires hc_pre_eps == hc_sinkhorn_eps (single merged hc_eps)"
|
||||
)
|
||||
assert hc_post_mult_value == 2.0, "mhc_pre_cpu hardcodes the post-mix multiplier"
|
||||
|
||||
hc_mult, hidden_size = residual.shape[-2:]
|
||||
outer_shape = residual.shape[:-2]
|
||||
x_flat = residual.reshape(-1, hc_mult, hidden_size)
|
||||
|
||||
layer_input, post, comb = ops.hc_pre_fused_cpu(
|
||||
x_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
hc_mult,
|
||||
sinkhorn_repeat,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
)
|
||||
|
||||
post_mix = post.view(*outer_shape, hc_mult, 1)
|
||||
comb_mix = comb.view(*outer_shape, hc_mult, hc_mult)
|
||||
layer_input = layer_input.view(*outer_shape, hidden_size)
|
||||
return post_mix, comb_mix, layer_input
|
||||
|
||||
|
||||
def mhc_post_cpu(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""CPU-ported mHC post block (see `mhc_post_torch` for the eager reference)."""
|
||||
hc_mult, hidden_size = residual.shape[-2:]
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
x_flat = x.reshape(-1, hidden_size)
|
||||
residual_flat = residual.reshape(-1, hc_mult, hidden_size)
|
||||
post_flat = post_layer_mix.reshape(-1, hc_mult).float()
|
||||
comb_flat = comb_res_mix.reshape(-1, hc_mult, hc_mult).float()
|
||||
|
||||
out = ops.hc_post_fused_cpu(x_flat, residual_flat, post_flat, comb_flat)
|
||||
return out.view(*outer_shape, hc_mult, hidden_size)
|
||||
|
||||
|
||||
def hc_head_fused_cpu(
|
||||
hidden_states: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
hc_eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""CPU-ported HC head reduction (see `test_hc_head_cpu` in
|
||||
tests/kernels/mhc/test_mhc_kernels.py for the eager reference this is tested
|
||||
against).
|
||||
|
||||
The ported kernel's C++ signature takes `(hc_eps, norm_eps)` -- the
|
||||
opposite order from this wrapper's `(rms_norm_eps, hc_eps)`, which matches
|
||||
the real call site in `models/deepseek_v4/cpu/model.py`.
|
||||
"""
|
||||
hc_mult, hidden_size = hidden_states.shape[-2:]
|
||||
outer_shape = hidden_states.shape[:-2]
|
||||
hs_flat = hidden_states.reshape(-1, hc_mult, hidden_size)
|
||||
|
||||
out = ops.hc_head_fused_cpu(hs_flat, hc_fn, hc_scale, hc_base, hc_eps, rms_norm_eps)
|
||||
return out.view(*outer_shape, hidden_size)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,147 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
|
||||
def mhc_pre_delayed_torch(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
x: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Reference for mHC pre using coefficients from the previous sublayer."""
|
||||
hc_mult = residual.shape[1]
|
||||
x = (residual.flatten(1) if x is None else x).float()
|
||||
mixes = (x @ fn.t()) * torch.rsqrt(x.square().mean(-1, keepdim=True) + rms_eps)
|
||||
pre = (
|
||||
torch.sigmoid(mixes[:, :hc_mult] * hc_scale[0] + hc_base[:hc_mult]) + hc_pre_eps
|
||||
)
|
||||
post = (
|
||||
torch.sigmoid(
|
||||
mixes[:, hc_mult : 2 * hc_mult] * hc_scale[1]
|
||||
+ hc_base[hc_mult : 2 * hc_mult]
|
||||
)
|
||||
* hc_post_mult_value
|
||||
)
|
||||
comb = mixes[:, 2 * hc_mult :].view(-1, hc_mult, hc_mult) * hc_scale[2]
|
||||
comb = comb + hc_base[2 * hc_mult :].view(1, hc_mult, hc_mult)
|
||||
comb = torch.softmax(comb, dim=-1) + hc_sinkhorn_eps
|
||||
comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
|
||||
for _ in range(sinkhorn_repeat - 1):
|
||||
comb = comb / (comb.sum(dim=-1, keepdim=True) + hc_sinkhorn_eps)
|
||||
comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
|
||||
layer_input = (
|
||||
residual[:, 0]
|
||||
if pre_mix is None
|
||||
else (pre_mix.unsqueeze(-1) * residual.float()).sum(dim=1).to(residual.dtype)
|
||||
)
|
||||
return post.unsqueeze(-1), comb, layer_input, pre
|
||||
|
||||
|
||||
def mhc_pre_torch(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Forward pass for mHC pre block.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
|
||||
hc_scale: shape (3,), dtype torch.float32
|
||||
hc_base: shape (hc_mult3,), dtype torch.float32
|
||||
rms_eps: RMS normalization epsilon
|
||||
hc_pre_eps: pre-mix epsilon
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
n_splits: split-k factor;
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
|
||||
"""
|
||||
# Validate shapes
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert fn.dtype == torch.float32
|
||||
assert hc_scale.dtype == torch.float32
|
||||
assert hc_base.dtype == torch.float32
|
||||
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult2 = hc_mult * hc_mult
|
||||
hc_mult3 = hc_mult * 2 + hc_mult2
|
||||
|
||||
hc_hidden_size = hc_mult * hidden_size
|
||||
assert fn.shape[0] == hc_mult3
|
||||
assert fn.shape[1] == hc_hidden_size
|
||||
assert hc_scale.shape == (3,)
|
||||
assert hc_base.shape == (hc_mult3,)
|
||||
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
fn_flat = fn
|
||||
|
||||
x = residual_flat.view(num_tokens, hc_mult * hidden_size).to(torch.float32)
|
||||
mixes = torch.matmul(x, fn_flat.t())
|
||||
sqrsum = x.square().sum(dim=-1, keepdim=True)
|
||||
mixes = mixes * torch.rsqrt(sqrsum / (hc_mult * hidden_size) + rms_eps)
|
||||
|
||||
pre_logits = mixes[:, :hc_mult] * hc_scale[0] + hc_base[:hc_mult]
|
||||
pre_mix = torch.sigmoid(pre_logits) + hc_pre_eps
|
||||
|
||||
post_logits = (
|
||||
mixes[:, hc_mult : 2 * hc_mult] * hc_scale[1] + hc_base[hc_mult : 2 * hc_mult]
|
||||
)
|
||||
post_mix = torch.sigmoid(post_logits) * hc_post_mult_value
|
||||
|
||||
comb_logits = mixes[:, 2 * hc_mult :].view(num_tokens, hc_mult, hc_mult) * hc_scale[
|
||||
2
|
||||
] + hc_base[2 * hc_mult :].view(1, hc_mult, hc_mult)
|
||||
comb_mix = torch.softmax(comb_logits, dim=-1) + hc_sinkhorn_eps
|
||||
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
|
||||
for _ in range(sinkhorn_repeat - 1):
|
||||
comb_mix = comb_mix / (comb_mix.sum(dim=-1, keepdim=True) + hc_sinkhorn_eps)
|
||||
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
|
||||
|
||||
layer_input = torch.sum(
|
||||
pre_mix.unsqueeze(-1) * residual_flat.to(torch.float32), dim=1
|
||||
).to(torch.bfloat16)
|
||||
return (
|
||||
post_mix.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix.view(*outer_shape, hc_mult, hc_mult),
|
||||
layer_input.view(*outer_shape, hidden_size),
|
||||
)
|
||||
|
||||
|
||||
def mhc_post_torch(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
mixed_residual = torch.einsum(
|
||||
"...ij,...ih->...jh",
|
||||
comb_res_mix.to(torch.float32),
|
||||
residual.to(torch.float32),
|
||||
)
|
||||
post_term = post_layer_mix.to(torch.float32) * x.unsqueeze(-2).to(torch.float32)
|
||||
return (mixed_residual + post_term).to(residual.dtype)
|
||||
@@ -1,349 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import Tensor
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rmsnorm_nw_kernel(
|
||||
x_ptr,
|
||||
out_ptr,
|
||||
stride_row,
|
||||
D,
|
||||
eps,
|
||||
RBLOCK: tl.constexpr,
|
||||
):
|
||||
"""Weight-free RMSNorm Triton kernel: out = x * rsqrt(mean(x², -1) + eps)."""
|
||||
row = tl.program_id(0)
|
||||
cols = tl.arange(0, RBLOCK)
|
||||
mask = cols < D
|
||||
|
||||
x = tl.load(
|
||||
x_ptr + row * stride_row + cols,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
eviction_policy="evict_first",
|
||||
).to(tl.float32)
|
||||
|
||||
var = tl.sum(x * x, 0) / D
|
||||
rstd = tl.rsqrt(var + eps)
|
||||
|
||||
out = (x * rstd).to(out_ptr.dtype.element_ty)
|
||||
tl.store(out_ptr + row * D + cols, out, mask=mask, eviction_policy="evict_first")
|
||||
|
||||
|
||||
def rmsnorm_nw(x: Tensor, eps: float) -> Tensor:
|
||||
"""Weight-free RMSNorm over the last dimension.
|
||||
|
||||
Treats *x* as ``[num_rows, D]`` where ``num_rows = product(shape[:-1])``.
|
||||
Returns a contiguous tensor with the same shape and dtype as *x*.
|
||||
"""
|
||||
orig_shape = x.shape
|
||||
D = orig_shape[-1]
|
||||
x_2d = x.reshape(-1, D)
|
||||
num_rows = x_2d.shape[0]
|
||||
|
||||
out = torch.empty_like(x_2d)
|
||||
RBLOCK = triton.next_power_of_2(D)
|
||||
|
||||
_rmsnorm_nw_kernel[(num_rows,)](
|
||||
x_2d,
|
||||
out,
|
||||
x_2d.stride(0),
|
||||
D,
|
||||
eps,
|
||||
RBLOCK=RBLOCK,
|
||||
num_warps=1 if RBLOCK <= 512 else (4 if RBLOCK <= 4096 else 8),
|
||||
)
|
||||
return out.view(orig_shape)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_head_reduce_store_kernel(
|
||||
pre_ptr,
|
||||
x_ptr,
|
||||
out_ptr,
|
||||
hidden_size: tl.constexpr,
|
||||
hc_mult: tl.constexpr,
|
||||
pre_stride_t: tl.constexpr,
|
||||
pre_stride_m: tl.constexpr,
|
||||
x_stride_t: tl.constexpr,
|
||||
x_stride_m: tl.constexpr,
|
||||
x_stride_h: tl.constexpr,
|
||||
out_stride_t: tl.constexpr,
|
||||
out_stride_h: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
block_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
mask = offsets < hidden_size
|
||||
|
||||
acc = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||||
for mix_idx in tl.static_range(0, hc_mult):
|
||||
pre = tl.load(pre_ptr + token_idx * pre_stride_t + mix_idx * pre_stride_m).to(
|
||||
tl.float32
|
||||
)
|
||||
x = tl.load(
|
||||
x_ptr
|
||||
+ token_idx * x_stride_t
|
||||
+ mix_idx * x_stride_m
|
||||
+ offsets * x_stride_h,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
acc += pre * x
|
||||
|
||||
tl.store(
|
||||
out_ptr + token_idx * out_stride_t + offsets * out_stride_h,
|
||||
acc,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
def hc_collapse_triton(x: Tensor, pre_mix: Tensor) -> Tensor:
|
||||
"""Collapse BF16 residual streams with FP32 pre-mix coefficients."""
|
||||
assert x.ndim == 3 and x.dtype == torch.bfloat16
|
||||
num_tokens, hc_mult, hidden_size = x.shape
|
||||
assert pre_mix.shape == (num_tokens, hc_mult)
|
||||
assert pre_mix.dtype == torch.float32
|
||||
out = torch.empty(num_tokens, hidden_size, dtype=x.dtype, device=x.device)
|
||||
if num_tokens == 0:
|
||||
return out
|
||||
|
||||
block_h = 1024
|
||||
_hc_head_reduce_store_kernel[(num_tokens, triton.cdiv(hidden_size, block_h))](
|
||||
pre_mix,
|
||||
x,
|
||||
out,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
pre_mix.stride(0),
|
||||
pre_mix.stride(1),
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
x.stride(2),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
BLOCK_H=block_h,
|
||||
num_warps=4,
|
||||
# Preserve the separate FP32 multiply and sum in the Torch reference.
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _hc_collapse_triton_fake(x: Tensor, pre_mix: Tensor) -> Tensor:
|
||||
return torch.empty(x.shape[0], x.shape[2], dtype=x.dtype, device=x.device)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mhc_pre_mix_kernel(
|
||||
gemm_ptr,
|
||||
sqrsum_ptr,
|
||||
hc_scale_ptr,
|
||||
hc_base_ptr,
|
||||
out_ptr,
|
||||
splitk,
|
||||
hc_mult: tl.constexpr,
|
||||
gemm_stride_k,
|
||||
gemm_stride_t,
|
||||
gemm_stride_j,
|
||||
sqrsum_stride_k,
|
||||
sqrsum_stride_t,
|
||||
out_stride_t,
|
||||
out_stride_j,
|
||||
inv_hc_hidden,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
SPLITK_BLOCK: tl.constexpr,
|
||||
HC_BLOCK: tl.constexpr,
|
||||
):
|
||||
"""Recover the pre-mix gate from a split-k mHC pre GEMM output."""
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
ks = tl.arange(0, SPLITK_BLOCK)
|
||||
js = tl.arange(0, HC_BLOCK)
|
||||
kmask = ks < splitk
|
||||
jmask = js < hc_mult
|
||||
|
||||
# Only the first hc_mult GEMM columns feed the pre gate.
|
||||
gemm = tl.load(
|
||||
gemm_ptr
|
||||
+ ks[:, None] * gemm_stride_k
|
||||
+ token_idx * gemm_stride_t
|
||||
+ js[None, :] * gemm_stride_j,
|
||||
mask=kmask[:, None] & jmask[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
mixes = tl.sum(gemm, 0)
|
||||
|
||||
sqrsum = tl.load(
|
||||
sqrsum_ptr + ks * sqrsum_stride_k + token_idx * sqrsum_stride_t,
|
||||
mask=kmask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
rstd = tl.rsqrt(tl.sum(sqrsum, 0) * inv_hc_hidden + rms_eps)
|
||||
|
||||
scale = tl.load(hc_scale_ptr).to(tl.float32)
|
||||
base = tl.load(hc_base_ptr + js, mask=jmask, other=0.0).to(tl.float32)
|
||||
|
||||
pre = tl.sigmoid(mixes * rstd * scale + base) + hc_pre_eps
|
||||
tl.store(out_ptr + token_idx * out_stride_t + js * out_stride_j, pre, mask=jmask)
|
||||
|
||||
|
||||
def mhc_pre_mix_triton(
|
||||
gemm_out: Tensor,
|
||||
sqrsum: Tensor,
|
||||
hc_scale: Tensor,
|
||||
hc_base: Tensor,
|
||||
hc_mult: int,
|
||||
hc_hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
) -> Tensor:
|
||||
"""Pre-mix gate for the delayed mHC pre, from AITER's split-k GEMM output.
|
||||
|
||||
AITER's ``mhc_pre_big_fuse`` consumes the unreduced ``[splitk, tokens,
|
||||
hc_mult3]`` GEMM output and the matching row square-sums, but only returns
|
||||
the post and comb gates. The delayed formulation also needs the pre gate,
|
||||
to carry into the next sublayer seam. It is the same slice of the same
|
||||
numbers, so recover it here rather than repeating the projection.
|
||||
"""
|
||||
assert gemm_out.ndim == 3 and gemm_out.dtype == torch.float32
|
||||
assert sqrsum.ndim == 2 and sqrsum.dtype == torch.float32
|
||||
splitk, num_tokens = gemm_out.shape[0], gemm_out.shape[1]
|
||||
assert sqrsum.shape == (splitk, num_tokens)
|
||||
|
||||
out = torch.empty(num_tokens, hc_mult, dtype=torch.float32, device=gemm_out.device)
|
||||
if num_tokens == 0:
|
||||
return out
|
||||
|
||||
_mhc_pre_mix_kernel[(num_tokens,)](
|
||||
gemm_out,
|
||||
sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
splitk,
|
||||
hc_mult,
|
||||
gemm_out.stride(0),
|
||||
gemm_out.stride(1),
|
||||
gemm_out.stride(2),
|
||||
sqrsum.stride(0),
|
||||
sqrsum.stride(1),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
1.0 / hc_hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
SPLITK_BLOCK=triton.next_power_of_2(splitk),
|
||||
HC_BLOCK=triton.next_power_of_2(hc_mult),
|
||||
num_warps=1,
|
||||
# Match the separate FP32 multiply and add in the Torch reference.
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _mhc_pre_mix_triton_fake(
|
||||
gemm_out: Tensor,
|
||||
sqrsum: Tensor,
|
||||
hc_scale: Tensor,
|
||||
hc_base: Tensor,
|
||||
hc_mult: int,
|
||||
hc_hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
) -> Tensor:
|
||||
return torch.empty(
|
||||
gemm_out.shape[1], hc_mult, dtype=torch.float32, device=gemm_out.device
|
||||
)
|
||||
|
||||
|
||||
def hc_head_reduce_triton_kernel(
|
||||
x: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
norm_eps: float,
|
||||
hc_eps: float,
|
||||
) -> None:
|
||||
x_flat = x.flatten(-2)
|
||||
x_normed = rmsnorm_nw(x_flat, norm_eps)
|
||||
mixes = F.linear(x_normed.float(), hc_fn)
|
||||
pre = torch.sigmoid(mixes * hc_scale + hc_base) + hc_eps
|
||||
|
||||
hidden_size = x.shape[-1]
|
||||
hc_mult = x.shape[-2]
|
||||
block_h = 1024
|
||||
_hc_head_reduce_store_kernel[(x.shape[0], (hidden_size + block_h - 1) // block_h)](
|
||||
pre,
|
||||
x,
|
||||
out,
|
||||
hidden_size,
|
||||
hc_mult,
|
||||
pre.stride(0),
|
||||
pre.stride(1),
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
x.stride(2),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
BLOCK_H=block_h,
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
|
||||
def _hc_head_triton(
|
||||
hs_flat: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
hc_mult: int,
|
||||
) -> None:
|
||||
"""Fill pre-allocated `out` (T, H) in-place with the hc_head result."""
|
||||
if hs_flat.shape[0] == 0:
|
||||
return
|
||||
|
||||
hc_head_reduce_triton_kernel(
|
||||
hs_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
rms_eps,
|
||||
hc_eps,
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="hc_head_triton",
|
||||
op_func=_hc_head_triton,
|
||||
mutates_args=["out"],
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="hc_collapse_triton",
|
||||
op_func=hc_collapse_triton,
|
||||
mutates_args=[],
|
||||
fake_impl=_hc_collapse_triton_fake,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre_mix_triton",
|
||||
op_func=mhc_pre_mix_triton,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_pre_mix_triton_fake,
|
||||
)
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
compute_num_split,
|
||||
mhc_pre_big_fuse_with_norm_tilelang,
|
||||
)
|
||||
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
|
||||
def compute_mhc_pre_num_splits(input_size: int, num_tokens: int) -> int:
|
||||
splits = compute_num_split(64, input_size, cdiv(num_tokens, 64))
|
||||
# Bound both GEMM and fused-normalization specializations during startup.
|
||||
return 1 if splits == 1 else 4 if splits <= 4 else 16
|
||||
|
||||
|
||||
class MHCPreNormKernel(VllmJitKernel["MHCPreNormKernel.CompileKey"]):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
hidden_size: int
|
||||
rms_eps: float
|
||||
hc_pre_eps: float
|
||||
hc_sinkhorn_eps: float
|
||||
hc_post_mult_value: float
|
||||
sinkhorn_repeat: int
|
||||
norm_eps: float
|
||||
n_splits: int
|
||||
hc_mult: int
|
||||
use_pre_mix_in: bool
|
||||
rms_numel: int
|
||||
save_pre_mix: bool = True
|
||||
split_mode: str = "fused"
|
||||
write_aux: bool = False
|
||||
|
||||
kernel: Any = staticmethod(mhc_pre_big_fuse_with_norm_tilelang)
|
||||
|
||||
def dispatch(self, *, n_splits, **fields) -> CompileKey: # type: ignore[override]
|
||||
return self.CompileKey(n_splits=n_splits, **fields)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
max_tokens: int,
|
||||
extra_splits: Iterable[int] = (),
|
||||
**fields,
|
||||
) -> list[CompileKey]:
|
||||
# The split heuristic changes only at 64-token boundaries. Callers that
|
||||
# reach this kernel through another GEMM (the fused post + pre-norm
|
||||
# GEMM) pass the splits that GEMM picks in extra_splits.
|
||||
splits = sorted(
|
||||
set(extra_splits)
|
||||
| (
|
||||
{
|
||||
compute_mhc_pre_num_splits(fields["rms_numel"], num_tokens)
|
||||
for num_tokens in range(1, max_tokens + 1, 64)
|
||||
}
|
||||
if is_deep_gemm_supported()
|
||||
else {1}
|
||||
)
|
||||
)
|
||||
return self._trace_dispatch(self.dispatch)(n_splits=splits, **fields)
|
||||
|
||||
def compile(self, compile_key: CompileKey) -> None:
|
||||
if compile_key not in self._compiled_cache:
|
||||
self._compiled_cache[compile_key] = self.kernel.compile(
|
||||
**asdict(compile_key)
|
||||
)
|
||||
|
||||
def compile_many(self, compile_keys: Iterable[CompileKey]) -> None:
|
||||
missing = list(
|
||||
dict.fromkeys(
|
||||
key for key in compile_keys if key not in self._compiled_cache
|
||||
)
|
||||
)
|
||||
if len(missing) < 2:
|
||||
return super().compile_many(missing)
|
||||
|
||||
# TileLang elaborates serially before compiling independent TIR modules.
|
||||
compiled = self.kernel.par_compile(
|
||||
[asdict(key) for key in missing], num_workers=min(4, len(missing))
|
||||
)
|
||||
self._compiled_cache.update(zip(missing, compiled, strict=True))
|
||||
|
||||
def __call__(self, *tensors, **fields):
|
||||
compile_key = self.dispatch(
|
||||
n_splits=tensors[0].shape[0],
|
||||
**fields,
|
||||
)
|
||||
if not current_platform.is_cuda():
|
||||
return self.kernel(*tensors, **asdict(compile_key))
|
||||
return self._get_or_compile(compile_key)(*tensors)
|
||||
|
||||
|
||||
MHC_PRE_NORM_KERNEL = MHCPreNormKernel()
|
||||
@@ -1,68 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
|
||||
from vllm.vllm_flash_attn.cute import utils
|
||||
|
||||
|
||||
@cute.jit
|
||||
def dense_mask_mod(
|
||||
batch: cute.TensorSSA,
|
||||
head: cute.TensorSSA,
|
||||
q_idx: cute.TensorSSA,
|
||||
kv_idx: cute.TensorSSA,
|
||||
seqlen_info,
|
||||
aux_tensors: list,
|
||||
) -> cute.TensorSSA:
|
||||
dense_mask = aux_tensors[0]
|
||||
batch_idx = utils.ssa_to_scalar(batch)
|
||||
q_idx = utils.ssa_to_scalar(q_idx)
|
||||
kv_idx = utils.ssa_to_scalar(kv_idx)
|
||||
batch_stride, query_stride, _ = dense_mask.stride
|
||||
aligned_mask = cute.make_tensor(
|
||||
dense_mask.iterator,
|
||||
cute.make_layout(
|
||||
dense_mask.shape,
|
||||
stride=(
|
||||
cute.assume(batch_stride, divby=4),
|
||||
cute.assume(query_stride, divby=4),
|
||||
1,
|
||||
),
|
||||
),
|
||||
)
|
||||
mask_row = aligned_mask[batch_idx, q_idx, None]
|
||||
mask_chunks = cute.flat_divide(mask_row, (4,))
|
||||
mask_chunk = mask_chunks[None, (kv_idx >> 5) >> 2]
|
||||
loaded = cute.make_rmem_tensor_like(mask_chunk)
|
||||
cute.autovec_copy(mask_chunk, loaded)
|
||||
return cute.recast_tensor(loaded, cutlass.Uint32).load()
|
||||
|
||||
|
||||
dense_mask_mod.__vec_size__ = 128
|
||||
|
||||
|
||||
@cute.jit
|
||||
def offset_dense_mask_mod(
|
||||
batch: cute.TensorSSA,
|
||||
head: cute.TensorSSA,
|
||||
q_idx: cute.TensorSSA,
|
||||
kv_idx: cute.TensorSSA,
|
||||
seqlen_info,
|
||||
aux_tensors: list,
|
||||
) -> cute.TensorSSA:
|
||||
dense_mask = aux_tensors[0]
|
||||
batch_idx = utils.ssa_to_scalar(batch)
|
||||
q_idx = utils.ssa_to_scalar(q_idx)
|
||||
key_start = dense_mask[batch_idx, 0, dense_mask.shape[2] - 1]
|
||||
kv_idx = utils.ssa_to_scalar(kv_idx) + key_start
|
||||
word_idx = kv_idx >> 5
|
||||
bit_idx = cutlass.Uint32(kv_idx & 31)
|
||||
word = dense_mask[batch_idx, q_idx, word_idx]
|
||||
result = cute.make_rmem_tensor(1, dtype=cutlass.Uint32)
|
||||
result[0] = utils.shr_u32(cutlass.Uint32(word), bit_idx)
|
||||
return result.load()
|
||||
|
||||
|
||||
offset_dense_mask_mod.__vec_size__ = 32
|
||||
@@ -119,11 +119,6 @@ if HAS_TRITON:
|
||||
TritonExperts,
|
||||
TritonWNA16Experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
|
||||
XPUExperts,
|
||||
XPUExpertsFp8,
|
||||
XPUExpertsMxFp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import (
|
||||
fused_experts,
|
||||
get_config_file_name,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,353 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_xpu():
|
||||
from vllm_xpu_kernels.fused_moe_interface import XpuFusedMoe
|
||||
|
||||
|
||||
def prepare_fp8_moe_layer_for_xpu(
|
||||
w13: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
if w13_scale is not None and w13_scale.ndim == 3:
|
||||
w13_scale = w13_scale.transpose(-1, -2).contiguous()
|
||||
if w2_scale is not None and w2_scale.ndim == 3:
|
||||
w2_scale = w2_scale.transpose(-1, -2).contiguous()
|
||||
return (
|
||||
w13.transpose(-1, -2).contiguous(),
|
||||
w13_scale,
|
||||
w2.transpose(-1, -2).contiguous(),
|
||||
w2_scale,
|
||||
)
|
||||
|
||||
|
||||
class XPUExperts(mk.FusedMoEExpertsModular):
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
moe_config,
|
||||
quant_config,
|
||||
max_num_tokens,
|
||||
num_dispatchers,
|
||||
)
|
||||
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
|
||||
self.fused_moe_impl: XpuFusedMoe | None = None
|
||||
is_xe2_or_xe3 = torch.ops._xpu_C.is_xe2_arch() or torch.ops._xpu_C.is_xe3_arch()
|
||||
if not is_xe2_or_xe3:
|
||||
raise NotImplementedError(
|
||||
"XPUExperts is only supported on Intel Xe2/Xe3 GPUs"
|
||||
)
|
||||
self._expects_unquantized_inputs = is_xe2_or_xe3
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return self._expects_unquantized_inputs
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_xpu()
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(None, None),
|
||||
(kFp8StaticTensorSym, None),
|
||||
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
workspace1 = (0,)
|
||||
workspace2 = (0,)
|
||||
output = (M, K)
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
if self.fused_moe_impl is None:
|
||||
topk = topk_ids.size(-1)
|
||||
if (
|
||||
self.quant_config is not None
|
||||
and self.quant_config.weight_quant_dtype == "mxfp4"
|
||||
):
|
||||
w1 = w1.view(torch.float4_e2m1fn_x2)
|
||||
w2 = w2.view(torch.float4_e2m1fn_x2)
|
||||
self.fused_moe_impl = XpuFusedMoe(
|
||||
w13=w1,
|
||||
w13_scales=self.w1_scale,
|
||||
w13_bias=self.w1_bias,
|
||||
w2=w2,
|
||||
w2_scales=self.w2_scale,
|
||||
w2_bias=self.w2_bias,
|
||||
n_experts_per_token=topk,
|
||||
activation=activation.value,
|
||||
num_experts=self.moe_config.num_local_experts,
|
||||
ep_rank=self.moe_config.ep_rank,
|
||||
ep_size=self.moe_config.ep_size,
|
||||
expert_map=expert_map,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
)
|
||||
assert self.fused_moe_impl is not None
|
||||
if self.moe_config.ep_size > 1:
|
||||
# Avoid topk_ids = -1, remap them to an
|
||||
# expert not owned by this rank so they are skipped.
|
||||
sentinel = global_num_experts - 1 if self.moe_config.ep_rank == 0 else 0
|
||||
topk_ids = torch.where(topk_ids < 0, sentinel, topk_ids)
|
||||
self.fused_moe_impl.apply(
|
||||
output=output,
|
||||
hidden_states=hidden_states,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
a1q_scale=a1q_scale,
|
||||
)
|
||||
|
||||
|
||||
class XPUExpertsFp8(XPUExperts):
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
moe_config,
|
||||
quant_config,
|
||||
max_num_tokens,
|
||||
num_dispatchers,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kFp8StaticTensorSym, None),
|
||||
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
class XPUExpertsMxFp8(XPUExpertsFp8):
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
moe_config,
|
||||
quant_config,
|
||||
max_num_tokens,
|
||||
num_dispatchers,
|
||||
)
|
||||
assert quant_config.quant_dtype == "mxfp8"
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kMxfp8Static, None),
|
||||
(kMxfp8Static, kMxfp8Dynamic),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
class XPUExpertsBlockFp8(XPUExperts):
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
moe_config,
|
||||
quant_config,
|
||||
max_num_tokens,
|
||||
num_dispatchers,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
class XPUExpertsWNA16(XPUExperts):
|
||||
"""W4A16 INT4-symmetric MoE backed by `xpu_fused_moe(is_int4=True)`.
|
||||
|
||||
Weight layout when `is_int4=True` (per `xpu_fused_moe` docstring):
|
||||
w13: [num_experts, 2*inter_size, hidden_size] contiguous int4-packed
|
||||
w13_scales: [num_experts, 2*inter_size, hidden_size // group_size]
|
||||
w2: [num_experts, hidden_size, inter_size] contiguous int4-packed
|
||||
w2_scales: [num_experts, hidden_size, inter_size // group_size]
|
||||
|
||||
Pairs with `INCXPULinearMethod` for the linear layers; together they
|
||||
cover full-attn + MoE on Intel XPU end-to-end without IPEX.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
moe_config,
|
||||
quant_config,
|
||||
max_num_tokens,
|
||||
num_dispatchers,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return (weight_key, activation_key) in (
|
||||
(kInt4Static, None),
|
||||
(kInt4Static32, None),
|
||||
)
|
||||
|
||||
|
||||
class XPUExpertsMxFp4(XPUExperts):
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
moe_config,
|
||||
quant_config,
|
||||
max_num_tokens,
|
||||
num_dispatchers,
|
||||
)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# K = a1q.size(-1). When activations are pre-quantized packed mxfp4,
|
||||
# K is the packed hidden_size (= logical / 2); the kernel output is at
|
||||
# logical hidden_size (2 * K). When unquantized (bf16), K is already
|
||||
# the logical size.
|
||||
logical_K = K if self.expects_unquantized_inputs else 2 * K
|
||||
return (0,), (0,), (M, logical_K)
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kMxfp4Static, None),
|
||||
(kMxfp4Static, kMxfp4Dynamic),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
@@ -241,21 +241,8 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [CutlassExpertsMxfp4]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.TRITON:
|
||||
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
|
||||
OAITritonExperts,
|
||||
OAITritonMxfp4ExpertsMonolithic,
|
||||
)
|
||||
|
||||
# NOTE: prefer Monolithic > Modular, so return Monolithic first.
|
||||
return [OAITritonMxfp4ExpertsMonolithic, OAITritonExperts]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.TRITON_UNFUSED:
|
||||
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
|
||||
UnfusedOAITritonExperts,
|
||||
)
|
||||
|
||||
return [UnfusedOAITritonExperts]
|
||||
elif backend in (Mxfp4MoeBackend.TRITON, Mxfp4MoeBackend.TRITON_UNFUSED):
|
||||
raise ImportError("gpt_oss triton_kernels experts were removed in this build.")
|
||||
|
||||
elif backend == Mxfp4MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
|
||||
@@ -1,213 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Producer side of the QuantizedActivation contract for activation layers.
|
||||
|
||||
Given an activation module and the downstream linear it feeds, fuse the
|
||||
activation with that linear's input quantization into a single kernel when the
|
||||
linear advertises a consumable input quantization key (see quant_activation.py).
|
||||
Falls back to the plain activation when nothing matches, so a model forward can
|
||||
always call maybe_fused_act_quant unconditionally.
|
||||
|
||||
This is the manual-fusion counterpart to ActivationQuantFusionPass: when fusion
|
||||
fires here, the activation and quantization are already consumed, so a compiler
|
||||
pass cannot fuse the same boundary again.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.activation import ReLUSquaredActivation, SiluAndMul
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.relu2_fp8_quant import (
|
||||
relu_squared_static_fp8_quant,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
FP4_DTYPE = torch.uint8
|
||||
|
||||
|
||||
def _silu_and_mul_fp8_static(
|
||||
x: torch.Tensor, linear: LinearBase
|
||||
) -> QuantizedActivation:
|
||||
"""SiluAndMul + FP8 static per-tensor quantization."""
|
||||
d = x.shape[-1] // 2
|
||||
out_shape = x.shape[:-1] + (d,)
|
||||
result = torch.empty(out_shape, dtype=FP8_DTYPE, device=x.device)
|
||||
# TODO(mgoin): read the consumer scale via the contract instead of reaching
|
||||
# into the kernel-specific input_scale attribute.
|
||||
scale = linear.input_scale
|
||||
torch.ops._C.silu_and_mul_quant(result, x, scale)
|
||||
return QuantizedActivation(
|
||||
data=result,
|
||||
scale=scale,
|
||||
orig_dtype=x.dtype,
|
||||
orig_shape=out_shape,
|
||||
quant_key=kFp8StaticTensorSym,
|
||||
)
|
||||
|
||||
|
||||
def _silu_and_mul_fp8_dynamic_block(
|
||||
x: torch.Tensor, linear: LinearBase, group_size: int, quant_key: QuantKey
|
||||
) -> QuantizedActivation:
|
||||
"""SiluAndMul + FP8 dynamic per-block quantization."""
|
||||
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
|
||||
d = x.shape[-1] // 2
|
||||
out_shape = x.shape[:-1] + (d,)
|
||||
x = x.reshape(-1, x.shape[-1])
|
||||
num_tokens = x.shape[0]
|
||||
num_groups = d // group_size
|
||||
|
||||
result = torch.empty((num_tokens, d), dtype=FP8_DTYPE, device=x.device)
|
||||
scales = torch.empty((num_tokens, num_groups), dtype=torch.float32, device=x.device)
|
||||
|
||||
torch.ops._C.silu_and_mul_per_block_quant(
|
||||
out=result,
|
||||
input=x,
|
||||
scales=scales,
|
||||
group_size=group_size,
|
||||
scale_ub=None,
|
||||
is_scale_transposed=False,
|
||||
)
|
||||
|
||||
return QuantizedActivation(
|
||||
data=result.view(out_shape),
|
||||
scale=scales.view(out_shape[:-1] + (num_groups,)),
|
||||
orig_dtype=x.dtype,
|
||||
orig_shape=out_shape,
|
||||
quant_key=quant_key,
|
||||
)
|
||||
|
||||
|
||||
def _silu_and_mul_fp8_dynamic_128(
|
||||
x: torch.Tensor, linear: LinearBase
|
||||
) -> QuantizedActivation:
|
||||
"""SiluAndMul + FP8 dynamic per-group (group=128) quantization."""
|
||||
return _silu_and_mul_fp8_dynamic_block(x, linear, 128, kFp8Dynamic128Sym)
|
||||
|
||||
|
||||
def _silu_and_mul_nvfp4_dynamic(
|
||||
x: torch.Tensor, linear: LinearBase
|
||||
) -> QuantizedActivation:
|
||||
"""SiluAndMul + NVFP4 dynamic quantization."""
|
||||
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
|
||||
d = x.shape[-1] // 2
|
||||
out_shape = x.shape[:-1] + (d,)
|
||||
x = x.reshape(-1, x.shape[-1])
|
||||
num_tokens = x.shape[0]
|
||||
|
||||
# NVFP4 packs 2 values into 1 byte
|
||||
result = torch.empty((num_tokens, d // 2), dtype=FP4_DTYPE, device=x.device)
|
||||
|
||||
# Block scale output: swizzled tensor-core layout
|
||||
# [num_m_tiles, num_k_tiles, 32, 4, 4] of int32-packed FP8 scales, so the
|
||||
# row/col extents must be padded to the 128x4 tile like
|
||||
# scaled_fp4_quant's allocator (create_fp4_scale_tensor). Each group of 16
|
||||
# elements shares one FP8 scale.
|
||||
rounded_m = round_up(num_tokens, 128)
|
||||
rounded_n = round_up(d // 16, 4)
|
||||
block_scale = torch.empty(
|
||||
(rounded_m, rounded_n // 4), dtype=torch.int32, device=x.device
|
||||
).view(FP8_DTYPE)
|
||||
|
||||
# The kernel folds the global scale into the block scales, and the
|
||||
# consumer GEMM's alpha (= input_global_scale * weight_global_scale)
|
||||
# divides it back out, so quantize with the reciprocal like the unfused
|
||||
# scaled_fp4_quant path does.
|
||||
input_global_scale_inv = getattr(linear, "input_global_scale_inv", None)
|
||||
assert input_global_scale_inv is not None, (
|
||||
"input_global_scale_inv is required for NVFP4 quantization"
|
||||
)
|
||||
|
||||
torch.ops._C.silu_and_mul_nvfp4_quant(
|
||||
result, block_scale, x, input_global_scale_inv
|
||||
)
|
||||
|
||||
return QuantizedActivation(
|
||||
data=result.view(out_shape[:-1] + (d // 2,)),
|
||||
scale=block_scale,
|
||||
orig_dtype=x.dtype,
|
||||
orig_shape=out_shape,
|
||||
quant_key=kNvfp4Dynamic,
|
||||
)
|
||||
|
||||
|
||||
# (activation module type, consumer input quantization key) -> fused producer.
|
||||
# Add a row for each supported manual producer.
|
||||
_FUSED_ACT_QUANT: dict[tuple[type, QuantKey], Callable] = {
|
||||
(SiluAndMul, kFp8StaticTensorSym): _silu_and_mul_fp8_static,
|
||||
}
|
||||
|
||||
|
||||
def _relu_squared_static_fp8_quant_supported(
|
||||
act_fn: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
linear: LinearBase,
|
||||
) -> bool:
|
||||
"""Return whether the ReLU2 static-FP8 producer can consume this input."""
|
||||
scale = getattr(linear, "input_scale", None)
|
||||
return (
|
||||
isinstance(act_fn, ReLUSquaredActivation)
|
||||
and x.is_cuda
|
||||
and x.dtype == torch.bfloat16
|
||||
and x.is_contiguous()
|
||||
and isinstance(scale, torch.Tensor)
|
||||
and scale.dtype == torch.float32
|
||||
and scale.device == x.device
|
||||
and scale.numel() == 1
|
||||
)
|
||||
|
||||
|
||||
# Optional per-entry predicates preserve the unconditional fallback contract
|
||||
# for producers whose supported inputs are narrower than their registry key.
|
||||
_FUSED_ACT_QUANT_SUPPORT: dict[
|
||||
tuple[type, QuantKey], Callable[[torch.nn.Module, torch.Tensor, LinearBase], bool]
|
||||
] = {}
|
||||
|
||||
# This path is validated on SM90 and newer CUDA devices.
|
||||
if current_platform.is_cuda() and current_platform.has_device_capability(90):
|
||||
_RELU2_STATIC_FP8_KEY = (ReLUSquaredActivation, kFp8StaticTensorSym)
|
||||
_FUSED_ACT_QUANT[_RELU2_STATIC_FP8_KEY] = relu_squared_static_fp8_quant
|
||||
_FUSED_ACT_QUANT_SUPPORT[_RELU2_STATIC_FP8_KEY] = (
|
||||
_relu_squared_static_fp8_quant_supported
|
||||
)
|
||||
|
||||
# Add CUDA-specific entries for dynamic block quantization
|
||||
if current_platform.is_cuda_alike():
|
||||
_FUSED_ACT_QUANT[(SiluAndMul, kFp8Dynamic128Sym)] = _silu_and_mul_fp8_dynamic_128
|
||||
|
||||
# Add NVFP4 if supported (requires SM100+)
|
||||
if current_platform.is_cuda() and hasattr(torch.ops._C, "silu_and_mul_nvfp4_quant"):
|
||||
_FUSED_ACT_QUANT[(SiluAndMul, kNvfp4Dynamic)] = _silu_and_mul_nvfp4_dynamic
|
||||
|
||||
|
||||
def maybe_fused_act_quant(
|
||||
act_fn: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
linear: LinearBase,
|
||||
) -> "torch.Tensor | QuantizedActivation":
|
||||
"""Apply act_fn, fusing the downstream linear's input quant when possible.
|
||||
|
||||
Returns a QuantizedActivation when a fused kernel matches the activation and
|
||||
the consumer's effective input quantization key, else the plain activation.
|
||||
"""
|
||||
key = get_input_quant_key(linear)
|
||||
if key is not None:
|
||||
registry_key = (type(act_fn), key)
|
||||
producer = _FUSED_ACT_QUANT.get(registry_key)
|
||||
support = _FUSED_ACT_QUANT_SUPPORT.get(registry_key)
|
||||
if producer is not None and (support is None or support(act_fn, x, linear)):
|
||||
return producer(x, linear)
|
||||
return act_fn(x)
|
||||
@@ -1,14 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointExporter,
|
||||
)
|
||||
|
||||
|
||||
def kda_prefill_checkpoint_alignment(backend: str) -> int | None:
|
||||
return 16 if backend == "flashkda" else None
|
||||
|
||||
|
||||
class FlashKDAPrefillCheckpointExporter(MambaPrefillCheckpointExporter):
|
||||
"""Store FlashKDA recurrent and convolution checkpoint states."""
|
||||
@@ -1,704 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.linear.base import LinearAttention
|
||||
from vllm.model_executor.layers.mamba.linear.minimax_linear_attn import (
|
||||
MiniMaxText01LinearAttention,
|
||||
MiniMaxText01LinearKernel,
|
||||
linear_attention_decode,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
|
||||
RMSNormGated,
|
||||
layernorm_fn,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
|
||||
|
||||
|
||||
def _build_rope_parameters(config: PreTrainedConfig) -> dict | None:
|
||||
rope_parameters = copy.deepcopy(getattr(config, "rope_parameters", None)) or {}
|
||||
if "rope_theta" not in rope_parameters and hasattr(config, "rope_theta"):
|
||||
rope_parameters["rope_theta"] = config.rope_theta
|
||||
if "partial_rotary_factor" not in rope_parameters and hasattr(
|
||||
config, "partial_rotary_factor"
|
||||
):
|
||||
rope_parameters["partial_rotary_factor"] = config.partial_rotary_factor
|
||||
|
||||
rope_scaling = getattr(config, "rope_scaling", None)
|
||||
if isinstance(rope_scaling, dict):
|
||||
rope_scaling = copy.deepcopy(rope_scaling)
|
||||
if "type" in rope_scaling and "rope_type" not in rope_scaling:
|
||||
rope_scaling["rope_type"] = rope_scaling.pop("type")
|
||||
rope_parameters.update(rope_scaling)
|
||||
|
||||
return rope_parameters or None
|
||||
|
||||
|
||||
def clear_linear_attention_cache_for_new_sequences(
|
||||
kv_cache: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
attn_metadata: LinearAttentionMetadata,
|
||||
) -> None:
|
||||
num_prefills = getattr(attn_metadata, "num_prefills", 0)
|
||||
if num_prefills <= 0:
|
||||
return
|
||||
|
||||
num_decodes = getattr(attn_metadata, "num_decodes", 0)
|
||||
prefill_state_indices = getattr(attn_metadata, "state_indices_tensor_p", None)
|
||||
for prefill_idx in range(num_prefills):
|
||||
if num_decodes + prefill_idx + 1 >= len(attn_metadata.query_start_loc):
|
||||
break
|
||||
q_start = attn_metadata.query_start_loc[num_decodes + prefill_idx]
|
||||
q_end = attn_metadata.query_start_loc[num_decodes + prefill_idx + 1]
|
||||
query_len = q_end - q_start
|
||||
context_len = attn_metadata.seq_lens[num_decodes + prefill_idx] - query_len
|
||||
if context_len == 0:
|
||||
if prefill_state_indices is not None:
|
||||
block_to_clear = prefill_state_indices[prefill_idx]
|
||||
else:
|
||||
block_to_clear = state_indices_tensor[num_decodes + prefill_idx]
|
||||
kv_cache[block_to_clear, ...] = 0
|
||||
|
||||
|
||||
def linear_attention_prefill_and_mix(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
attn_metadata: LinearAttentionMetadata,
|
||||
slope_rate: torch.Tensor,
|
||||
block_size: int,
|
||||
decode_fn,
|
||||
prefix_fn,
|
||||
layer_idx: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
hidden = []
|
||||
req_offset = getattr(attn_metadata, "num_decodes", 0)
|
||||
prefill_state_indices = getattr(attn_metadata, "state_indices_tensor_p", None)
|
||||
for _prefill_idx in range(getattr(attn_metadata, "num_prefills", 0)):
|
||||
if req_offset + _prefill_idx + 1 >= len(attn_metadata.query_start_loc):
|
||||
break
|
||||
if prefill_state_indices is not None and _prefill_idx >= len(
|
||||
prefill_state_indices
|
||||
):
|
||||
break
|
||||
if prefill_state_indices is None and _prefill_idx >= len(state_indices_tensor):
|
||||
break
|
||||
_start = attn_metadata.query_start_loc[req_offset + _prefill_idx]
|
||||
_end = attn_metadata.query_start_loc[req_offset + _prefill_idx + 1]
|
||||
if prefill_state_indices is not None:
|
||||
slot_id = prefill_state_indices[_prefill_idx]
|
||||
else:
|
||||
slot_id = state_indices_tensor[req_offset + _prefill_idx]
|
||||
qs = q[_start:_end].transpose(0, 1).contiguous()
|
||||
ks = k[_start:_end].transpose(0, 1).contiguous()
|
||||
vs = v[_start:_end].transpose(0, 1).contiguous()
|
||||
slice_layer_cache = kv_cache[slot_id, ...]
|
||||
out_slice = prefix_fn(
|
||||
qs,
|
||||
ks,
|
||||
vs,
|
||||
slice_layer_cache,
|
||||
slope_rate,
|
||||
block_size,
|
||||
layer_idx=layer_idx,
|
||||
)
|
||||
hidden.append(out_slice.contiguous())
|
||||
|
||||
if attn_metadata.num_decode_tokens > 0:
|
||||
hidden_decode = decode_fn(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
hidden.insert(0, hidden_decode)
|
||||
|
||||
if not hidden:
|
||||
return torch.empty((0, q.size(1) * q.size(2)), device=q.device, dtype=q.dtype)
|
||||
|
||||
hidden = torch.concat(hidden, dim=0).contiguous()
|
||||
return hidden
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _bailing_linear_attn_decode_spec_step_kernel(
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
v_ptr,
|
||||
kv_cache_ptr,
|
||||
slope_rate_ptr,
|
||||
state_indices_ptr,
|
||||
query_start_loc_ptr,
|
||||
num_accepted_tokens_ptr,
|
||||
output_ptr,
|
||||
q_start: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
q_b_stride,
|
||||
q_h_stride,
|
||||
q_d_stride,
|
||||
k_b_stride,
|
||||
k_h_stride,
|
||||
k_d_stride,
|
||||
v_b_stride,
|
||||
v_h_stride,
|
||||
v_d_stride,
|
||||
cache_b_stride,
|
||||
cache_h_stride,
|
||||
cache_d0_stride,
|
||||
cache_d1_stride,
|
||||
state_indices_b_stride,
|
||||
state_indices_t_stride,
|
||||
output_b_stride,
|
||||
output_d_stride,
|
||||
DRAFT_IDX: tl.constexpr,
|
||||
STATE_WIDTH: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
req_id = tl.program_id(0)
|
||||
head_id = tl.program_id(1)
|
||||
block_id = tl.program_id(2)
|
||||
|
||||
req_start = tl.load(query_start_loc_ptr + req_id).to(tl.int64)
|
||||
req_end = tl.load(query_start_loc_ptr + req_id + 1).to(tl.int64)
|
||||
query_len = req_end - req_start
|
||||
if query_len <= DRAFT_IDX:
|
||||
return
|
||||
|
||||
dst_slot = tl.load(
|
||||
state_indices_ptr
|
||||
+ req_id * state_indices_b_stride
|
||||
+ DRAFT_IDX * state_indices_t_stride
|
||||
).to(tl.int64)
|
||||
if dst_slot == -1:
|
||||
return
|
||||
|
||||
if DRAFT_IDX == 0:
|
||||
accepted_offset = tl.load(num_accepted_tokens_ptr + req_id).to(tl.int64) - 1
|
||||
accepted_offset = tl.maximum(accepted_offset, 0)
|
||||
accepted_offset = tl.minimum(accepted_offset, STATE_WIDTH - 1)
|
||||
src_slot = tl.load(
|
||||
state_indices_ptr
|
||||
+ req_id * state_indices_b_stride
|
||||
+ accepted_offset * state_indices_t_stride
|
||||
).to(tl.int64)
|
||||
else:
|
||||
src_slot = tl.load(
|
||||
state_indices_ptr
|
||||
+ req_id * state_indices_b_stride
|
||||
+ (DRAFT_IDX - 1) * state_indices_t_stride
|
||||
).to(tl.int64)
|
||||
if src_slot == -1:
|
||||
return
|
||||
|
||||
token_idx = req_start - q_start + DRAFT_IDX
|
||||
qk_offsets = tl.arange(0, D)
|
||||
v_offsets = tl.arange(0, BLOCK_SIZE) + block_id * BLOCK_SIZE
|
||||
qk_mask = qk_offsets < D
|
||||
v_mask = v_offsets < D
|
||||
kv_mask = qk_mask[:, None] & v_mask[None, :]
|
||||
|
||||
q = tl.load(
|
||||
q_ptr + token_idx * q_b_stride + head_id * q_h_stride + qk_offsets * q_d_stride,
|
||||
mask=qk_mask,
|
||||
other=0.0,
|
||||
)
|
||||
k = tl.load(
|
||||
k_ptr + token_idx * k_b_stride + head_id * k_h_stride + qk_offsets * k_d_stride,
|
||||
mask=qk_mask,
|
||||
other=0.0,
|
||||
)
|
||||
v = tl.load(
|
||||
v_ptr + token_idx * v_b_stride + head_id * v_h_stride + v_offsets * v_d_stride,
|
||||
mask=v_mask,
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
cache_offsets = (
|
||||
qk_offsets[:, None] * cache_d0_stride + v_offsets[None, :] * cache_d1_stride
|
||||
)
|
||||
src_cache_ptr = (
|
||||
kv_cache_ptr
|
||||
+ src_slot * cache_b_stride
|
||||
+ head_id * cache_h_stride
|
||||
+ cache_offsets
|
||||
)
|
||||
dst_cache_ptr = (
|
||||
kv_cache_ptr
|
||||
+ dst_slot * cache_b_stride
|
||||
+ head_id * cache_h_stride
|
||||
+ cache_offsets
|
||||
)
|
||||
|
||||
slope = tl.load(slope_rate_ptr + head_id)
|
||||
decay = tl.exp(-slope)
|
||||
kv_old = tl.load(src_cache_ptr, mask=kv_mask, other=0.0)
|
||||
kv_new = k[:, None] * v[None, :] + decay * kv_old
|
||||
|
||||
output = tl.sum(q[:, None].to(tl.float32) * kv_new, axis=0)
|
||||
tl.store(dst_cache_ptr, kv_new, mask=kv_mask)
|
||||
tl.store(
|
||||
output_ptr
|
||||
+ token_idx * output_b_stride
|
||||
+ (head_id * D + v_offsets) * output_d_stride,
|
||||
output,
|
||||
mask=v_mask,
|
||||
)
|
||||
|
||||
|
||||
def bailing_linear_attention_decode_spec(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slope_rate: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
q_start: int,
|
||||
q_end: int | None,
|
||||
slot_start: int,
|
||||
slot_end: int | None,
|
||||
block_size: int,
|
||||
) -> torch.Tensor:
|
||||
q_decode = q[q_start:q_end]
|
||||
k_decode = k[q_start:q_end]
|
||||
v_decode = v[q_start:q_end]
|
||||
hidden = torch.empty(
|
||||
(q_decode.shape[0], q.shape[1] * q.shape[2]),
|
||||
device=q.device,
|
||||
dtype=q.dtype,
|
||||
)
|
||||
hidden.zero_()
|
||||
|
||||
state_indices_tensor = state_indices_tensor[slot_start:slot_end]
|
||||
query_start_loc = query_start_loc.to(device=q.device)
|
||||
|
||||
batch_size = state_indices_tensor.shape[0]
|
||||
num_heads = q_decode.shape[1]
|
||||
head_dim = q_decode.shape[2]
|
||||
assert k_decode.shape == (q_decode.shape[0], num_heads, head_dim)
|
||||
assert v_decode.shape == (q_decode.shape[0], num_heads, head_dim)
|
||||
state_width = state_indices_tensor.shape[1]
|
||||
|
||||
grid = (batch_size, num_heads, triton.cdiv(head_dim, block_size))
|
||||
for draft_idx in range(state_width):
|
||||
_bailing_linear_attn_decode_spec_step_kernel[grid](
|
||||
q_decode,
|
||||
k_decode,
|
||||
v_decode,
|
||||
kv_cache,
|
||||
slope_rate,
|
||||
state_indices_tensor,
|
||||
query_start_loc,
|
||||
num_accepted_tokens[:batch_size],
|
||||
hidden,
|
||||
q_start,
|
||||
head_dim,
|
||||
q_decode.stride(0),
|
||||
q_decode.stride(1),
|
||||
q_decode.stride(2),
|
||||
k_decode.stride(0),
|
||||
k_decode.stride(1),
|
||||
k_decode.stride(2),
|
||||
v_decode.stride(0),
|
||||
v_decode.stride(1),
|
||||
v_decode.stride(2),
|
||||
kv_cache.stride(0),
|
||||
kv_cache.stride(1),
|
||||
kv_cache.stride(2),
|
||||
kv_cache.stride(3),
|
||||
state_indices_tensor.stride(0),
|
||||
state_indices_tensor.stride(1),
|
||||
hidden.stride(0),
|
||||
hidden.stride(1),
|
||||
DRAFT_IDX=draft_idx,
|
||||
STATE_WIDTH=state_width,
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
|
||||
return hidden
|
||||
|
||||
|
||||
class BailingGroupRMSNormGate(RMSNormGated):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size,
|
||||
eps=1e-5,
|
||||
group_size=None,
|
||||
norm_before_gate=True,
|
||||
device=None,
|
||||
dtype=None,
|
||||
):
|
||||
super().__init__(
|
||||
hidden_size,
|
||||
eps=eps,
|
||||
group_size=group_size,
|
||||
norm_before_gate=norm_before_gate,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
activation="sigmoid",
|
||||
)
|
||||
# Add custom weight loader for TP sharding
|
||||
self.weight.weight_loader = self._weight_loader
|
||||
|
||||
@staticmethod
|
||||
def _weight_loader(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None:
|
||||
"""Load weight with TP sharding."""
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
shard_size = loaded_weight.shape[0] // tp_size
|
||||
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
|
||||
param.data.copy_(loaded_weight[shard].contiguous())
|
||||
|
||||
|
||||
# --8<-- [start:bailing_moe_linear_attention]
|
||||
@PluggableLayer.register("bailing_moe_linear_attention")
|
||||
class BailingMoELinearAttention(LinearAttention):
|
||||
"""Pluggable Bailing MoE Linear Attention layer which allows OOT backends
|
||||
to add custom implementations.
|
||||
|
||||
This implements the linear attention mechanism from sglang, adapted for
|
||||
vLLM's v1 engine with MambaBase interface support.
|
||||
"""
|
||||
|
||||
# --8<-- [end:bailing_moe_linear_attention]
|
||||
def __init__(
|
||||
self,
|
||||
config: PreTrainedConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "linear_attn",
|
||||
):
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
|
||||
self.scaling = self.head_dim**-0.5
|
||||
|
||||
self.tp_heads = self.num_heads // self.tp_size
|
||||
|
||||
self.max_position_embeddings = config.max_position_embeddings
|
||||
self.rope_theta = getattr(config, "rope_theta", 600000)
|
||||
|
||||
self.tp_kv_heads = self.num_heads // self.tp_size
|
||||
self.q_size_per_rank = self.head_dim * self.tp_heads
|
||||
self.kv_size_per_rank = self.head_dim * self.tp_kv_heads
|
||||
|
||||
self.use_qk_norm = getattr(config, "use_qk_norm", False)
|
||||
self.linear_backend = "minimax"
|
||||
self.linear_scale = self.linear_backend == "minimax"
|
||||
self.linear_rope = getattr(config, "linear_rope", True)
|
||||
if hasattr(config, "use_linear_silu"):
|
||||
self.linear_silu = config.use_linear_silu
|
||||
elif hasattr(config, "linear_silu"):
|
||||
self.linear_silu = config.linear_silu
|
||||
else:
|
||||
self.linear_silu = False
|
||||
|
||||
self.query_key_value = QKVParallelLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
self.num_heads,
|
||||
self.num_heads, # MHA: kv_heads = num_heads
|
||||
bias=(config.use_bias or config.use_qkv_bias),
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.query_key_value",
|
||||
)
|
||||
|
||||
if self.use_qk_norm:
|
||||
self.query_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.key_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
self.g_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_inner_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_proj",
|
||||
)
|
||||
self.dense = RowParallelLinear(
|
||||
self.hidden_inner_size,
|
||||
self.hidden_size,
|
||||
bias=config.use_bias,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.dense",
|
||||
reduce_results=True,
|
||||
)
|
||||
|
||||
self.group_norm_size = getattr(config, "group_norm_size", 1)
|
||||
self.rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
|
||||
assert self.tp_size <= self.group_norm_size, (
|
||||
"tp_size must be <= group_norm_size for local rms norm"
|
||||
)
|
||||
assert self.group_norm_size % self.tp_size == 0, (
|
||||
"group_norm_size must be divisible by tp_size"
|
||||
)
|
||||
|
||||
# When group_norm_size == 1, group_size equals hidden_size // tp_size
|
||||
self.g_norm = BailingGroupRMSNormGate(
|
||||
hidden_size=self.hidden_inner_size // self.tp_size,
|
||||
eps=self.rms_norm_eps,
|
||||
group_size=(
|
||||
self.hidden_inner_size // self.group_norm_size
|
||||
if self.group_norm_size > 1
|
||||
else self.hidden_inner_size // self.tp_size
|
||||
),
|
||||
)
|
||||
|
||||
# use fp32 rotary embedding
|
||||
rope_parameters = _build_rope_parameters(config)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=self.max_position_embeddings,
|
||||
is_neox_style=True,
|
||||
rope_parameters=rope_parameters or None,
|
||||
)
|
||||
|
||||
# Build slope tensor for linear attention decay
|
||||
slope_rate = MiniMaxText01LinearAttention._build_slope_tensor(self.num_heads)
|
||||
if self.num_hidden_layers <= 1:
|
||||
self.slope_rate = slope_rate * (1 + 1e-5)
|
||||
else:
|
||||
self.slope_rate = slope_rate * (
|
||||
1 - self.layer_idx / (self.num_hidden_layers - 1) + 1e-5
|
||||
)
|
||||
self.tp_slope = self.slope_rate[
|
||||
self.tp_rank * self.tp_heads : (self.tp_rank + 1) * self.tp_heads
|
||||
].contiguous()
|
||||
|
||||
# Register for compilation
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def get_attn_backend(self):
|
||||
from vllm.v1.attention.backends.linear_attn import (
|
||||
BailingLinearAttentionBackend,
|
||||
)
|
||||
|
||||
return BailingLinearAttentionBackend
|
||||
|
||||
@staticmethod
|
||||
def weight_direct_load(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
"""Load weight for linear attention layers.
|
||||
|
||||
For FP8 quantized parameters, we need to use the weight_loader if available,
|
||||
as it handles special cases like tensor parallelism sharding.
|
||||
"""
|
||||
# Check if param has a weight_loader (for vLLM ModelWeightParameter)
|
||||
weight_loader = getattr(param, "weight_loader", None)
|
||||
if weight_loader is not None:
|
||||
# Use the weight_loader which handles TP sharding and quantization
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
# Fall back to direct copy for standard tensors
|
||||
assert param.size() == loaded_weight.size(), (
|
||||
f"Shape mismatch: {param.shape} vs {loaded_weight.shape}"
|
||||
)
|
||||
param.data.copy_(loaded_weight)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
"""Forward method called by torch.ops.vllm.linear_attention."""
|
||||
torch.ops.vllm.linear_attention(
|
||||
hidden_states,
|
||||
output,
|
||||
positions,
|
||||
self.prefix,
|
||||
)
|
||||
|
||||
def _forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
"""Actual forward implementation."""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, dict)
|
||||
attn_metadata = attn_metadata.get(self.prefix) # type: ignore
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, LinearAttentionMetadata)
|
||||
num_actual_tokens = (
|
||||
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
|
||||
)
|
||||
else:
|
||||
num_actual_tokens = hidden_states.shape[0]
|
||||
|
||||
# QKV projection
|
||||
qkv, _ = self.query_key_value(hidden_states[:num_actual_tokens])
|
||||
|
||||
# use rotary_emb support fp32
|
||||
qkv = qkv.to(torch.float32)
|
||||
if self.linear_silu:
|
||||
qkv = F.silu(qkv)
|
||||
|
||||
# Split q, k, v
|
||||
q, k, v = torch.split(
|
||||
qkv,
|
||||
[self.q_size_per_rank, self.kv_size_per_rank, self.kv_size_per_rank],
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
# Apply QK norm if needed
|
||||
if self.use_qk_norm:
|
||||
q = q.reshape(-1, self.tp_heads, self.head_dim)
|
||||
k = k.reshape(-1, self.tp_kv_heads, self.head_dim)
|
||||
q = layernorm_fn(
|
||||
q,
|
||||
self.query_layernorm.weight.data,
|
||||
bias=None,
|
||||
eps=self.rms_norm_eps,
|
||||
is_rms_norm=True,
|
||||
)
|
||||
k = layernorm_fn(
|
||||
k,
|
||||
self.key_layernorm.weight.data,
|
||||
bias=None,
|
||||
eps=self.rms_norm_eps,
|
||||
is_rms_norm=True,
|
||||
)
|
||||
q = q.reshape(-1, self.q_size_per_rank)
|
||||
k = k.reshape(-1, self.kv_size_per_rank)
|
||||
|
||||
# Apply rotary embeddings
|
||||
if self.linear_rope:
|
||||
q, k = self.rotary_emb(positions[:num_actual_tokens], q, k)
|
||||
|
||||
# Reshape to [batch, heads, seq_len, head_dim]
|
||||
q = q.view((qkv.shape[0], self.tp_heads, self.head_dim))
|
||||
k = k.view((qkv.shape[0], self.tp_kv_heads, self.head_dim))
|
||||
v = v.view((qkv.shape[0], self.tp_kv_heads, self.head_dim))
|
||||
|
||||
# Apply scaling if using minimax backend
|
||||
if self.linear_scale:
|
||||
q = q * self.scaling
|
||||
|
||||
# Get KV cache and state indices
|
||||
if attn_metadata is not None:
|
||||
kv_cache = self.kv_cache[0]
|
||||
state_indices_tensor = attn_metadata.state_indices_tensor
|
||||
clear_linear_attention_cache_for_new_sequences(
|
||||
kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
|
||||
# Compute attention
|
||||
decode_only = getattr(attn_metadata, "num_prefills", 0) == 0
|
||||
if attn_metadata is None:
|
||||
hidden = torch.empty(
|
||||
(q.shape[0], q.shape[1] * q.shape[2]), device=q.device, dtype=q.dtype
|
||||
)
|
||||
else:
|
||||
if not decode_only:
|
||||
hidden = self._prefill_and_mix_infer(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
else:
|
||||
hidden = self._decode_infer(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
|
||||
# Apply group norm and gate (matching SGLang behavior)
|
||||
gate, _ = self.g_proj(hidden_states[:num_actual_tokens])
|
||||
|
||||
if self.group_norm_size > 1:
|
||||
hidden = self.g_norm(hidden, gate)
|
||||
else:
|
||||
hidden = self.g_norm(hidden)
|
||||
hidden = F.sigmoid(gate) * hidden
|
||||
|
||||
hidden = hidden.to(hidden_states.dtype)
|
||||
|
||||
# Output projection
|
||||
dense_out, _ = self.dense(hidden)
|
||||
output[:num_actual_tokens] = dense_out
|
||||
|
||||
def _prefill_and_mix_infer(
|
||||
self, q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
):
|
||||
"""Handle prefill (mixed with decode if any)."""
|
||||
return linear_attention_prefill_and_mix(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
kv_cache=kv_cache,
|
||||
state_indices_tensor=state_indices_tensor,
|
||||
attn_metadata=attn_metadata,
|
||||
slope_rate=self.tp_slope,
|
||||
block_size=self.BLOCK,
|
||||
decode_fn=self._decode_infer,
|
||||
prefix_fn=MiniMaxText01LinearKernel.jit_linear_forward_prefix,
|
||||
layer_idx=self.layer_idx,
|
||||
)
|
||||
|
||||
def _decode_infer(self, q, k, v, kv_cache, state_indices_tensor, attn_metadata):
|
||||
"""Handle decode (single token per sequence)."""
|
||||
decode_state_indices = getattr(attn_metadata, "state_indices_tensor_d", None)
|
||||
num_accepted_tokens = getattr(attn_metadata, "num_accepted_tokens", None)
|
||||
query_start_loc = getattr(attn_metadata, "query_start_loc_d", None)
|
||||
if (
|
||||
decode_state_indices is not None
|
||||
and decode_state_indices.dim() > 1
|
||||
and num_accepted_tokens is not None
|
||||
and query_start_loc is not None
|
||||
):
|
||||
return bailing_linear_attention_decode_spec(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
kv_cache,
|
||||
self.tp_slope,
|
||||
decode_state_indices,
|
||||
query_start_loc,
|
||||
num_accepted_tokens,
|
||||
q_start=0,
|
||||
q_end=attn_metadata.num_decode_tokens,
|
||||
slot_start=0,
|
||||
slot_end=attn_metadata.num_decodes,
|
||||
block_size=32,
|
||||
)
|
||||
decode_state_indices = (
|
||||
state_indices_tensor
|
||||
if decode_state_indices is None
|
||||
else decode_state_indices
|
||||
)
|
||||
hidden = linear_attention_decode(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
kv_cache,
|
||||
self.tp_slope,
|
||||
decode_state_indices,
|
||||
q_start=0,
|
||||
q_end=attn_metadata.num_decode_tokens,
|
||||
slot_start=0,
|
||||
slot_end=attn_metadata.num_decodes,
|
||||
block_size=32,
|
||||
)
|
||||
return hidden
|
||||
@@ -1,66 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.mamba.abstract import MambaBase
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
|
||||
|
||||
class LinearAttention(PluggableLayer, MambaBase):
|
||||
"""Base class for Linear attention layer."""
|
||||
|
||||
def __init__(
|
||||
self, config: PreTrainedConfig, vllm_config: VllmConfig, prefix: str = ""
|
||||
):
|
||||
super().__init__()
|
||||
self.layer_idx = extract_layer_index(prefix)
|
||||
self.prefix = prefix
|
||||
self.model_config = vllm_config.model_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
self.BLOCK = getattr(config, "block", 256)
|
||||
self.hidden_size = config.hidden_size
|
||||
self.num_heads = config.num_attention_heads
|
||||
self.num_hidden_layers = config.num_hidden_layers
|
||||
self.head_dim = (
|
||||
config.head_dim
|
||||
if hasattr(config, "head_dim")
|
||||
else config.hidden_size // self.num_heads
|
||||
)
|
||||
self.hidden_inner_size = self.head_dim * self.num_heads
|
||||
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
assert self.num_heads % self.tp_size == 0
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.LINEAR
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype]:
|
||||
assert self.model_config is not None
|
||||
assert self.cache_config is not None
|
||||
return MambaStateDtypeCalculator.linear_attention_state_dtype(
|
||||
self.model_config.dtype,
|
||||
self.cache_config.mamba_cache_dtype,
|
||||
)
|
||||
|
||||
def get_state_shape(self) -> tuple[tuple[int, int, int], ...]:
|
||||
return MambaStateShapeCalculator.linear_attention_state_shape(
|
||||
num_heads=self.num_heads, tp_size=self.tp_size, head_dim=self.head_dim
|
||||
)
|
||||
@@ -1,345 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.lightning_attn import (
|
||||
lightning_attention,
|
||||
linear_decode_forward_triton,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.mamba.linear.base import LinearAttention
|
||||
from vllm.model_executor.layers.minimax_rms_norm import MiniMaxText01RMSNormTP
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
|
||||
|
||||
|
||||
def clear_linear_attention_cache_for_new_sequences(
|
||||
kv_cache: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
attn_metadata: LinearAttentionMetadata,
|
||||
) -> None:
|
||||
num_prefills = getattr(attn_metadata, "num_prefills", 0)
|
||||
if num_prefills <= 0:
|
||||
return
|
||||
|
||||
num_decode_tokens = getattr(attn_metadata, "num_decode_tokens", 0)
|
||||
for prefill_idx in range(num_prefills):
|
||||
q_start = attn_metadata.query_start_loc[num_decode_tokens + prefill_idx]
|
||||
q_end = attn_metadata.query_start_loc[num_decode_tokens + prefill_idx + 1]
|
||||
query_len = q_end - q_start
|
||||
context_len = (
|
||||
attn_metadata.seq_lens[num_decode_tokens + prefill_idx] - query_len
|
||||
)
|
||||
if context_len == 0:
|
||||
block_to_clear = state_indices_tensor[num_decode_tokens + prefill_idx]
|
||||
kv_cache[block_to_clear, ...] = 0
|
||||
|
||||
|
||||
def linear_attention_decode(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slope_rate: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
q_start: int = 0,
|
||||
q_end: int | None = None,
|
||||
slot_start: int = 0,
|
||||
slot_end: int | None = None,
|
||||
block_size: int = 32,
|
||||
) -> torch.Tensor:
|
||||
q = q[q_start:q_end].unsqueeze(2).contiguous()
|
||||
k = k[q_start:q_end].unsqueeze(2).contiguous()
|
||||
v = v[q_start:q_end].unsqueeze(2).contiguous()
|
||||
slot_id = state_indices_tensor[slot_start:slot_end]
|
||||
return linear_decode_forward_triton(
|
||||
q, k, v, kv_cache, slope_rate, slot_id, block_size
|
||||
)
|
||||
|
||||
|
||||
def linear_attention_prefill_and_mix(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
attn_metadata: LinearAttentionMetadata,
|
||||
slope_rate: torch.Tensor,
|
||||
block_size: int,
|
||||
decode_fn: Callable[..., torch.Tensor],
|
||||
prefix_fn: Callable[..., torch.Tensor],
|
||||
layer_idx: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
hidden = []
|
||||
for _prefill_idx in range(getattr(attn_metadata, "num_prefills", 0)):
|
||||
if _prefill_idx >= len(attn_metadata.query_start_loc):
|
||||
break
|
||||
if _prefill_idx >= len(state_indices_tensor):
|
||||
break
|
||||
offset = attn_metadata.num_decode_tokens
|
||||
_start = attn_metadata.query_start_loc[offset + _prefill_idx]
|
||||
_end = attn_metadata.query_start_loc[offset + _prefill_idx + 1]
|
||||
slot_id = state_indices_tensor[offset + _prefill_idx]
|
||||
qs = q[_start:_end].transpose(0, 1).contiguous()
|
||||
ks = k[_start:_end].transpose(0, 1).contiguous()
|
||||
vs = v[_start:_end].transpose(0, 1).contiguous()
|
||||
slice_layer_cache = kv_cache[slot_id, ...]
|
||||
out_slice = prefix_fn(
|
||||
qs,
|
||||
ks,
|
||||
vs,
|
||||
slice_layer_cache,
|
||||
slope_rate,
|
||||
block_size,
|
||||
layer_idx=layer_idx,
|
||||
)
|
||||
hidden.append(out_slice.contiguous())
|
||||
|
||||
if attn_metadata.num_decode_tokens > 0:
|
||||
hidden_decode = decode_fn(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
hidden.insert(0, hidden_decode)
|
||||
|
||||
if not hidden:
|
||||
return torch.empty((0, q.size(-1)), device=q.device, dtype=q.dtype)
|
||||
|
||||
hidden = torch.concat(hidden, dim=0).contiguous()
|
||||
return hidden
|
||||
|
||||
|
||||
class MiniMaxText01LinearKernel:
|
||||
@staticmethod
|
||||
def jit_linear_forward_prefix(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_caches: torch.Tensor,
|
||||
slope_rate: torch.Tensor,
|
||||
block_size: int,
|
||||
layer_idx: int | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
slope_rate = slope_rate.to(torch.float32)
|
||||
should_pad_dim = q.dim() == 3
|
||||
if should_pad_dim:
|
||||
q = q.unsqueeze(0)
|
||||
k = k.unsqueeze(0)
|
||||
v = v.unsqueeze(0)
|
||||
b, h, n, d = q.shape
|
||||
e = d
|
||||
kv_history = kv_caches.reshape(1, h, d, e).contiguous()
|
||||
output, kv_history = lightning_attention(
|
||||
q, k, v, slope_rate, block_size=block_size, kv_history=kv_history
|
||||
)
|
||||
kv_caches.copy_(kv_history[:, :, -1, :, :].reshape(h, d, e))
|
||||
assert output.shape[0] == 1, "batch size must be 1"
|
||||
return rearrange(output.squeeze(0), "h n d -> n (h d)")
|
||||
|
||||
|
||||
@PluggableLayer.register("minimax_text_01_attention")
|
||||
class MiniMaxText01LinearAttention(LinearAttention):
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
vllm_config,
|
||||
prefix: str = "linear_attn",
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
|
||||
self.tp_heads = self.num_heads // self.tp_size
|
||||
self.qkv_size = self.num_heads * self.head_dim
|
||||
self.tp_hidden = self.head_dim * self.tp_heads
|
||||
|
||||
self.qkv_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_inner_size * 3,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.output_gate = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_inner_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.output_gate",
|
||||
)
|
||||
self.out_proj = RowParallelLinear(
|
||||
self.hidden_inner_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.out_proj",
|
||||
)
|
||||
self.norm = MiniMaxText01RMSNormTP(
|
||||
self.hidden_inner_size,
|
||||
eps=1e-5,
|
||||
)
|
||||
|
||||
slope_rate = MiniMaxText01LinearAttention._build_slope_tensor(self.num_heads)
|
||||
if self.num_hidden_layers <= 1:
|
||||
self.slope_rate = slope_rate * (1 + 1e-5)
|
||||
else:
|
||||
self.slope_rate = slope_rate * (
|
||||
1 - self.layer_idx / (self.num_hidden_layers - 1) + 1e-5
|
||||
)
|
||||
self.tp_slope = self.slope_rate[
|
||||
self.tp_rank * self.tp_heads : (self.tp_rank + 1) * self.tp_heads
|
||||
].contiguous()
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
@staticmethod
|
||||
def weight_direct_load(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
assert param.size() == loaded_weight.size()
|
||||
param.data.copy_(loaded_weight)
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def _build_slope_tensor(n_attention_heads: int):
|
||||
def get_slopes(n):
|
||||
def get_slopes_power_of_2(n):
|
||||
start = 2 ** (-(2 ** -(math.log2(n) - 3)))
|
||||
ratio = start
|
||||
return [start * ratio**i for i in range(n)]
|
||||
|
||||
if math.log2(n).is_integer():
|
||||
return get_slopes_power_of_2(n)
|
||||
else:
|
||||
closest_power_of_2 = 2 ** math.floor(math.log2(n))
|
||||
return (
|
||||
get_slopes_power_of_2(closest_power_of_2)
|
||||
+ get_slopes(2 * closest_power_of_2)[0::2][: n - closest_power_of_2]
|
||||
)
|
||||
|
||||
slopes = torch.tensor(
|
||||
get_slopes(n_attention_heads), dtype=torch.float32
|
||||
).reshape(n_attention_heads, 1, 1)
|
||||
return slopes
|
||||
|
||||
def _prefill_and_mix_infer(
|
||||
self, q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
):
|
||||
return linear_attention_prefill_and_mix(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
kv_cache=kv_cache,
|
||||
state_indices_tensor=state_indices_tensor,
|
||||
attn_metadata=attn_metadata,
|
||||
slope_rate=self.tp_slope,
|
||||
block_size=self.BLOCK,
|
||||
decode_fn=self._decode_infer,
|
||||
prefix_fn=MiniMaxText01LinearKernel.jit_linear_forward_prefix,
|
||||
layer_idx=self.layer_idx,
|
||||
)
|
||||
|
||||
def _decode_infer(self, q, k, v, kv_cache, state_indices_tensor, attn_metadata):
|
||||
hidden = linear_attention_decode(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
kv_cache,
|
||||
self.tp_slope,
|
||||
state_indices_tensor,
|
||||
q_start=0,
|
||||
q_end=attn_metadata.num_decode_tokens,
|
||||
slot_start=0,
|
||||
slot_end=attn_metadata.num_decodes,
|
||||
block_size=32,
|
||||
)
|
||||
return hidden
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, output: torch.Tensor, positions: torch.Tensor
|
||||
) -> None:
|
||||
torch.ops.vllm.linear_attention(
|
||||
hidden_states,
|
||||
output,
|
||||
positions,
|
||||
self.prefix,
|
||||
)
|
||||
|
||||
def _forward(
|
||||
self, hidden_states: torch.Tensor, output: torch.Tensor, positions: torch.Tensor
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, LinearAttentionMetadata)
|
||||
num_actual_tokens = (
|
||||
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
|
||||
)
|
||||
else:
|
||||
num_actual_tokens = hidden_states.shape[0]
|
||||
|
||||
qkv, _ = self.qkv_proj(hidden_states[:num_actual_tokens])
|
||||
qkv32 = qkv.to(torch.float32)
|
||||
qkvact = torch.nn.functional.silu(qkv32)
|
||||
qkvact = qkvact.view((qkv.shape[0], self.tp_heads, -1))
|
||||
q, k, v = torch.split(qkvact, [self.head_dim] * 3, dim=-1)
|
||||
if attn_metadata is not None:
|
||||
kv_cache = self.kv_cache[0]
|
||||
state_indices_tensor = attn_metadata.state_indices_tensor
|
||||
clear_linear_attention_cache_for_new_sequences(
|
||||
kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
|
||||
decode_only = getattr(attn_metadata, "num_prefills", 0) == 0
|
||||
if attn_metadata is None:
|
||||
hidden = torch.empty(
|
||||
(q.shape[0], q.shape[1] * q.shape[2]), device=q.device, dtype=q.dtype
|
||||
)
|
||||
else:
|
||||
if not decode_only:
|
||||
hidden = self._prefill_and_mix_infer(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
else:
|
||||
hidden = self._decode_infer(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
hidden = self.norm._forward(hidden)
|
||||
gate, _ = self.output_gate(hidden_states[:num_actual_tokens])
|
||||
hidden = F.sigmoid(gate) * hidden
|
||||
hidden = hidden.to(hidden_states.dtype)
|
||||
|
||||
output[:num_actual_tokens], _ = self.out_proj(hidden)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def linear_attention(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._forward(hidden_states=hidden_states, output=output, positions=positions)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="linear_attention",
|
||||
op_func=linear_attention,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
@@ -1,485 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.abstract import MambaBase
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_scan_fn
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import selective_state_update
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
|
||||
|
||||
# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
|
||||
# --8<-- [start:mamba_mixer]
|
||||
@PluggableLayer.register("mamba_mixer")
|
||||
class MambaMixer(MambaBase, PluggableLayer):
|
||||
"""Compute ∆, A, B, C, and D the state space parameters and compute
|
||||
the `contextualized_states`. A, D are input independent
|
||||
(see Mamba paper [1] Section 3.5.2 "Interpretation of A"
|
||||
for why A isn't selective) ∆, B, C are input-dependent
|
||||
(this is a key difference between Mamba and the linear time
|
||||
invariant S4, and is why Mamba is called
|
||||
**selective** state spaces)
|
||||
"""
|
||||
|
||||
# --8<-- [end:mamba_mixer]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
ssm_state_size: int,
|
||||
conv_kernel_size: int,
|
||||
intermediate_size: int,
|
||||
time_step_rank: int,
|
||||
use_conv_bias: bool,
|
||||
use_bias: bool,
|
||||
use_rms_norm: bool,
|
||||
rms_norm_has_weight: bool = True,
|
||||
rms_norm_eps: float = 1e-5,
|
||||
activation="silu",
|
||||
is_lora_enabled: bool = False,
|
||||
model_config: ModelConfig | None = None,
|
||||
cache_config: CacheConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.time_step_rank = time_step_rank
|
||||
self.ssm_state_size = ssm_state_size
|
||||
self.use_rms_norm = use_rms_norm
|
||||
self.activation = activation
|
||||
self.is_lora_enabled = is_lora_enabled
|
||||
self.conv_kernel_size = conv_kernel_size
|
||||
self.intermediate_size = intermediate_size
|
||||
|
||||
self.conv1d = ColumnParallelLinear(
|
||||
input_size=conv_kernel_size,
|
||||
output_size=intermediate_size,
|
||||
bias=use_conv_bias,
|
||||
prefix=f"{prefix}.conv1d",
|
||||
)
|
||||
# unsqueeze to fit conv1d weights shape into the linear weights shape.
|
||||
# Can't do this in `weight_loader` since it already exists in
|
||||
# `ColumnParallelLinear` and `set_weight_attrs`
|
||||
# doesn't allow to override it
|
||||
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||
|
||||
self.in_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=use_bias,
|
||||
prefix=f"{prefix}.in_proj",
|
||||
)
|
||||
|
||||
# selective projection used to make dt, B and C input dependent
|
||||
self.x_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
time_step_rank + ssm_state_size * 2,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.x_proj",
|
||||
)
|
||||
# time step projection (discretization) -
|
||||
# In the forward we need to apply dt_proj without the bias,
|
||||
# as the bias is added in the selective scan kernel.
|
||||
self.dt_proj = ColumnParallelLinear(
|
||||
time_step_rank,
|
||||
intermediate_size,
|
||||
bias=True,
|
||||
skip_bias_add=True,
|
||||
prefix=f"{prefix}.dt_proj",
|
||||
)
|
||||
|
||||
def weight_loader(param: Parameter, loaded_weight: torch.Tensor):
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
param.data.copy_(
|
||||
loaded_weight.data.split(loaded_weight.shape[0] // tp_size, dim=0)[
|
||||
tp_rank
|
||||
]
|
||||
)
|
||||
|
||||
def A_weight_loader(param: Parameter, loaded_weight: torch.Tensor):
|
||||
weight_loader(param, -torch.exp(loaded_weight.float()))
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.A = nn.Parameter(
|
||||
torch.empty(
|
||||
intermediate_size // tp_size,
|
||||
ssm_state_size,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
)
|
||||
self.D = nn.Parameter(torch.ones(intermediate_size // tp_size))
|
||||
|
||||
set_weight_attrs(self.D, {"weight_loader": weight_loader})
|
||||
set_weight_attrs(self.A, {"weight_loader": A_weight_loader})
|
||||
|
||||
self.out_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=use_bias,
|
||||
input_is_parallel=True,
|
||||
prefix=f"{prefix}.out_proj",
|
||||
)
|
||||
|
||||
self.dt_layernorm = (
|
||||
RMSNorm(
|
||||
time_step_rank,
|
||||
eps=rms_norm_eps,
|
||||
has_weight=rms_norm_has_weight,
|
||||
)
|
||||
if use_rms_norm
|
||||
else None
|
||||
)
|
||||
|
||||
self.b_layernorm = (
|
||||
RMSNorm(
|
||||
ssm_state_size,
|
||||
eps=rms_norm_eps,
|
||||
has_weight=rms_norm_has_weight,
|
||||
)
|
||||
if use_rms_norm
|
||||
else None
|
||||
)
|
||||
|
||||
self.c_layernorm = (
|
||||
RMSNorm(
|
||||
ssm_state_size,
|
||||
eps=rms_norm_eps,
|
||||
has_weight=rms_norm_has_weight,
|
||||
)
|
||||
if use_rms_norm
|
||||
else None
|
||||
)
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
# The inner tuple is (conv_state, ssm_state)
|
||||
self.kv_cache = (torch.tensor([]), torch.tensor([]))
|
||||
|
||||
self.model_config = model_config
|
||||
self.cache_config = cache_config
|
||||
self.prefix = prefix
|
||||
|
||||
def _ssm_transform(
|
||||
self, x: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# LoRA kernel requires contiguous tensor.
|
||||
# ROCm: Non-contiguous tensors cause incorrect GEMM
|
||||
# results when batch > 1.
|
||||
if self.is_lora_enabled or current_platform.is_rocm():
|
||||
x = x.contiguous()
|
||||
ssm_params = self.x_proj(x)[0]
|
||||
time_step, B, C = torch.split(
|
||||
ssm_params,
|
||||
[self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
|
||||
dim=-1,
|
||||
)
|
||||
if self.use_rms_norm:
|
||||
assert self.dt_layernorm is not None
|
||||
assert self.b_layernorm is not None
|
||||
assert self.c_layernorm is not None
|
||||
time_step = self.dt_layernorm(time_step.contiguous())
|
||||
B = self.b_layernorm(B.contiguous())
|
||||
C = self.c_layernorm(C.contiguous())
|
||||
|
||||
# ROCm: tensor from split is non-contiguous, causing incorrect
|
||||
# GEMM results in dt_proj.
|
||||
if current_platform.is_rocm():
|
||||
time_step = time_step.contiguous()
|
||||
|
||||
discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
|
||||
return discrete_time_step, B, C
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor, output: torch.Tensor):
|
||||
torch.ops.vllm.mamba_mixer(
|
||||
hidden_states,
|
||||
output,
|
||||
_encode_layer_name(self.prefix),
|
||||
)
|
||||
|
||||
def forward_impl(self, hidden_states: torch.Tensor, output: torch.Tensor):
|
||||
"""Run the Mamba-1 SSM pipeline.
|
||||
|
||||
Steps
|
||||
-----
|
||||
1. Apply the gated-MLP linear projection to the raw input.
|
||||
2. Pass the projected sequence through the convolutional mixing layer.
|
||||
3. Feed the result into the State-Space Model (SSM) blocks.
|
||||
4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
|
||||
to produce contextual representations.
|
||||
5. Project the contextualised sequence back
|
||||
to the output embedding dimension.
|
||||
|
||||
Batch handling
|
||||
--------------
|
||||
Prefill and decode tokens are processed by dedicated CUDA
|
||||
kernels for both the convolutional (conv1d) and SSM stages.
|
||||
In the case of a mixed batch (containing both prefill and
|
||||
decode tokens), both sets of kernels are executed independently
|
||||
and their outputs are concatenated before the final output projection.
|
||||
"""
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
assert self.cache_config is not None
|
||||
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, Mamba1AttentionMetadata)
|
||||
query_start_loc_p = attn_metadata.query_start_loc_p
|
||||
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
|
||||
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
|
||||
conv_state = (
|
||||
self.kv_cache[0]
|
||||
if is_conv_state_dim_first()
|
||||
else self.kv_cache[0].transpose(-1, -2)
|
||||
)
|
||||
ssm_state = self.kv_cache[1]
|
||||
has_initial_states_p = attn_metadata.has_initial_states_p
|
||||
cu_chunk_seqlen_p = attn_metadata.cu_chunk_seqlen_p
|
||||
last_chunk_indices_p = attn_metadata.last_chunk_indices_p
|
||||
|
||||
# 1. Gated MLP's linear projection
|
||||
projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
|
||||
hidden_states_BC, gate = projected_states.chunk(2, dim=-2)
|
||||
|
||||
conv_weights = self.conv1d.weight.view(
|
||||
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||
)
|
||||
|
||||
if attn_metadata is None:
|
||||
# V1 profile run
|
||||
hidden_states_BC = hidden_states_BC.contiguous()
|
||||
return self.out_proj(hidden_states_BC.transpose(-2, -1))[0]
|
||||
|
||||
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
has_prefill = num_prefill_tokens > 0
|
||||
has_decode = num_decode_tokens > 0
|
||||
num_actual_tokens = num_prefill_tokens + num_decode_tokens
|
||||
|
||||
prefill_decode_split = split_batch_to_prefill_and_decode(
|
||||
hidden_states_BC,
|
||||
gate,
|
||||
num_prefill_tokens,
|
||||
num_decode_tokens,
|
||||
)
|
||||
hidden_states_BC_p = prefill_decode_split.hidden_states_BC_p
|
||||
hidden_states_BC_d = prefill_decode_split.hidden_states_BC_d
|
||||
gate_p = prefill_decode_split.gate_p
|
||||
gate_d = prefill_decode_split.gate_d
|
||||
|
||||
ssm_outputs = []
|
||||
|
||||
if has_prefill:
|
||||
# 2. Convolution sequence transformation
|
||||
conv_out_p = causal_conv1d_fn(
|
||||
hidden_states_BC_p,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation=self.activation,
|
||||
conv_states=conv_state,
|
||||
has_initial_state=has_initial_states_p,
|
||||
cache_indices=state_indices_tensor_p,
|
||||
query_start_loc=query_start_loc_p,
|
||||
metadata=attn_metadata,
|
||||
)
|
||||
# 3. State Space Model sequence transformations.
|
||||
discrete_time_step_p, B_p, C_p = self._ssm_transform(
|
||||
conv_out_p.transpose(-2, -1)
|
||||
)
|
||||
time_proj_bias = self._time_proj_bias()
|
||||
|
||||
# 4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
|
||||
scan_out_p = selective_scan_fn(
|
||||
conv_out_p,
|
||||
ssm_state,
|
||||
discrete_time_step_p,
|
||||
self.A,
|
||||
B_p.transpose(-2, -1),
|
||||
C_p.transpose(-2, -1),
|
||||
self.D.float(),
|
||||
gate_p,
|
||||
time_proj_bias,
|
||||
delta_softplus=True,
|
||||
cache_indices=state_indices_tensor_p,
|
||||
has_initial_state=has_initial_states_p,
|
||||
query_start_loc=query_start_loc_p,
|
||||
cu_chunk_seqlen=cu_chunk_seqlen_p,
|
||||
last_chunk_indices=last_chunk_indices_p,
|
||||
)
|
||||
ssm_outputs.append(scan_out_p)
|
||||
|
||||
if has_decode:
|
||||
# state_indices_tensor_d is assigned when attn_metadata is not None,
|
||||
# and has_decode is only True when attn_metadata is not None
|
||||
assert state_indices_tensor_d is not None
|
||||
# 2. Convolution sequence transformation
|
||||
conv_out_d = causal_conv1d_update(
|
||||
hidden_states_BC_d.transpose(0, 1),
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
self.activation,
|
||||
conv_state_indices=state_indices_tensor_d,
|
||||
).transpose(0, 1)
|
||||
|
||||
# 3. State Space Model sequence transformation.
|
||||
discrete_time_step_d, B_d, C_d = self._ssm_transform(
|
||||
conv_out_d.transpose(-2, -1)
|
||||
)
|
||||
time_proj_bias = self._time_proj_bias()
|
||||
|
||||
# 4. Perform the recurrence y ← SSM(A, B, C, Δ)(x)
|
||||
scan_outputs_d = torch.empty_like(hidden_states_BC_d.transpose(0, 1))
|
||||
selective_state_update(
|
||||
ssm_state,
|
||||
conv_out_d.transpose(0, 1),
|
||||
discrete_time_step_d.transpose(0, 1),
|
||||
self.A,
|
||||
B_d,
|
||||
C_d,
|
||||
self.D,
|
||||
time_proj_bias,
|
||||
z=gate_d.transpose(0, 1),
|
||||
dt_softplus=True,
|
||||
state_batch_indices=state_indices_tensor_d,
|
||||
dst_state_batch_indices=state_indices_tensor_d,
|
||||
out=scan_outputs_d,
|
||||
)
|
||||
scan_outputs_d = scan_outputs_d.transpose(0, 1)
|
||||
|
||||
ssm_outputs.insert(0, scan_outputs_d)
|
||||
|
||||
scan_outputs_combined = (
|
||||
ssm_outputs[0] if len(ssm_outputs) == 1 else torch.cat(ssm_outputs, dim=-1)
|
||||
)
|
||||
|
||||
# 5. Final output projection
|
||||
if self.is_lora_enabled: # Lora kernel requires contiguous tensor.
|
||||
scan_outputs_combined = scan_outputs_combined.transpose(-2, -1).contiguous()
|
||||
out = self.out_proj(scan_outputs_combined)[0]
|
||||
else:
|
||||
out = self.out_proj(scan_outputs_combined.transpose(-2, -1))[0]
|
||||
|
||||
output[:num_actual_tokens] = out
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype]:
|
||||
assert self.model_config is not None
|
||||
assert self.cache_config is not None
|
||||
return MambaStateDtypeCalculator.mamba1_state_dtype(
|
||||
self.model_config.dtype,
|
||||
self.cache_config.mamba_cache_dtype,
|
||||
self.cache_config.mamba_ssm_cache_dtype,
|
||||
)
|
||||
|
||||
def get_state_shape(self) -> tuple[tuple[int, ...], tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.mamba1_state_shape(
|
||||
tp_world_size=get_tensor_model_parallel_world_size(),
|
||||
intermediate_size=self.intermediate_size,
|
||||
state_size=self.ssm_state_size,
|
||||
conv_kernel=self.conv_kernel_size,
|
||||
)
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.MAMBA1
|
||||
|
||||
def _time_proj_bias(self) -> torch.Tensor | None:
|
||||
if hasattr(self.dt_proj, "bias") and self.dt_proj.bias is not None:
|
||||
return self.dt_proj.bias.float()
|
||||
return None
|
||||
|
||||
|
||||
class PrefillDecodeSplit(NamedTuple):
|
||||
hidden_states_BC_p: torch.Tensor
|
||||
hidden_states_BC_d: torch.Tensor
|
||||
gate_p: torch.Tensor
|
||||
gate_d: torch.Tensor
|
||||
|
||||
|
||||
def split_batch_to_prefill_and_decode(
|
||||
hidden_states_BC: torch.Tensor,
|
||||
gate: torch.Tensor,
|
||||
num_prefill_tokens: int,
|
||||
num_decode_tokens: int,
|
||||
) -> PrefillDecodeSplit:
|
||||
num_actual_tokens = num_prefill_tokens + num_decode_tokens
|
||||
|
||||
# In v1, decode tokens come first, then prefill tokens.
|
||||
hidden_states_BC_d, hidden_states_BC_p = torch.split(
|
||||
hidden_states_BC[..., :num_actual_tokens],
|
||||
[num_decode_tokens, num_prefill_tokens],
|
||||
dim=-1,
|
||||
)
|
||||
gate_d, gate_p = torch.split(
|
||||
gate[..., :num_actual_tokens], [num_decode_tokens, num_prefill_tokens], dim=-1
|
||||
)
|
||||
|
||||
return PrefillDecodeSplit(
|
||||
hidden_states_BC_p=hidden_states_BC_p,
|
||||
hidden_states_BC_d=hidden_states_BC_d,
|
||||
gate_p=gate_p,
|
||||
gate_d=gate_d,
|
||||
)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def mamba_mixer(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self.forward_impl(hidden_states=hidden_states, output=output)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mamba_mixer",
|
||||
op_func=mamba_mixer,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
@@ -1,83 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_initial_states_kernel(
|
||||
state_ptr,
|
||||
indices_ptr,
|
||||
has_initial_state_ptr,
|
||||
output_ptr,
|
||||
stride_state_batch,
|
||||
stride_indices,
|
||||
stride_has_initial_state,
|
||||
row_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
block_idx = tl.program_id(0)
|
||||
batch_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < row_size
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
has_initial_state = tl.load(
|
||||
has_initial_state_ptr + batch_idx * stride_has_initial_state
|
||||
).to(tl.int1)
|
||||
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
|
||||
state_idx = tl.where(has_initial_state, state_idx, 0)
|
||||
values = tl.load(
|
||||
state_ptr + state_idx * stride_state_batch + offsets,
|
||||
mask=mask & has_initial_state,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(output_ptr + batch_idx * row_size + offsets, values, mask=mask)
|
||||
|
||||
|
||||
def gather_initial_states(
|
||||
state: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
has_initial_state: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Gather dense state rows, replacing uninitialized rows with zeros."""
|
||||
assert state.ndim >= 2
|
||||
assert state.is_cuda or state.is_xpu
|
||||
assert indices.ndim == 1 and has_initial_state.ndim == 1
|
||||
assert indices.shape == has_initial_state.shape
|
||||
assert indices.device == state.device
|
||||
assert has_initial_state.device == state.device
|
||||
assert indices.dtype in (torch.int32, torch.int64)
|
||||
assert has_initial_state.dtype == torch.bool
|
||||
|
||||
row_size = state[0].numel()
|
||||
# Mamba pages may pad stride(0), but each state row remains dense.
|
||||
assert state[0].is_contiguous()
|
||||
output = torch.empty(
|
||||
(indices.numel(), *state.shape[1:]),
|
||||
dtype=state.dtype,
|
||||
device=state.device,
|
||||
)
|
||||
block_size = min(triton.next_power_of_2(row_size), 1024)
|
||||
grid = (triton.cdiv(row_size, block_size), indices.numel())
|
||||
_gather_initial_states_kernel[grid](
|
||||
state,
|
||||
indices,
|
||||
has_initial_state,
|
||||
output,
|
||||
state.stride(0),
|
||||
indices.stride(0),
|
||||
has_initial_state.stride(0),
|
||||
row_size=row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=8,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
return output
|
||||
@@ -1,105 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import WarmupIntRange
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
DispatchSpec,
|
||||
TritonWarmupTensor,
|
||||
triton_kernel_dispatcher_with_warmup,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _scatter_states_kernel(
|
||||
state_ptr,
|
||||
src_ptr,
|
||||
indices_ptr,
|
||||
stride_state_batch,
|
||||
stride_src_batch,
|
||||
stride_indices,
|
||||
row_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
block_idx = tl.program_id(0)
|
||||
batch_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < row_size
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
state_idx = tl.load(indices_ptr + batch_idx * stride_indices).to(tl.int64)
|
||||
values = tl.load(src_ptr + batch_idx * stride_src_batch + offsets, mask=mask)
|
||||
tl.store(state_ptr + state_idx * stride_state_batch + offsets, values, mask=mask)
|
||||
|
||||
|
||||
def _scatter_states_warmup_inputs(
|
||||
*,
|
||||
state_shape: tuple[int, ...],
|
||||
state_dtype: torch.dtype,
|
||||
indices_dtype: torch.dtype,
|
||||
max_num_tokens: int,
|
||||
) -> dict[str, object]:
|
||||
num_tokens: Any = WarmupIntRange(1, max_num_tokens + 1)
|
||||
return dict(
|
||||
state=TritonWarmupTensor(
|
||||
state_dtype,
|
||||
shape=(1,) + state_shape,
|
||||
),
|
||||
src=TritonWarmupTensor(
|
||||
state_dtype,
|
||||
shape=(num_tokens,) + state_shape,
|
||||
),
|
||||
indices=TritonWarmupTensor(
|
||||
indices_dtype,
|
||||
shape=(num_tokens,),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=_scatter_states_kernel,
|
||||
warmup_inputs=_scatter_states_warmup_inputs,
|
||||
)
|
||||
def scatter_states(
|
||||
state: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
) -> DispatchSpec:
|
||||
"""Scatter ``src`` rows into ``state`` at ``indices`` (in place).
|
||||
|
||||
Equivalent to ``state[indices] = src`` but non-atomic and bandwidth-bound,
|
||||
since mamba cache slots are unique per sequence. ``gather_initial_states``
|
||||
is the read-side counterpart.
|
||||
"""
|
||||
assert state.ndim >= 2
|
||||
assert state.is_cuda
|
||||
assert src.ndim == state.ndim
|
||||
assert indices.ndim == 1
|
||||
assert indices.device == state.device
|
||||
assert src.shape[1:] == state.shape[1:]
|
||||
assert src.shape[0] == indices.shape[0]
|
||||
assert indices.dtype in (torch.int32, torch.int64)
|
||||
|
||||
row_size = state[0].numel()
|
||||
assert state[0].is_contiguous()
|
||||
assert src[0].is_contiguous()
|
||||
block_size = min(triton.next_power_of_2(row_size), 1024)
|
||||
grid = (triton.cdiv(row_size, block_size), indices.numel())
|
||||
return grid, dict(
|
||||
stride_state_batch=state.stride(0),
|
||||
stride_src_batch=src.stride(0),
|
||||
stride_indices=indices.stride(0),
|
||||
row_size=row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=8,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,5 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .abstract import *
|
||||
from .common import *
|
||||
from .special import *
|
||||
@@ -1,37 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Set
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.outputs import PoolerOutput
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .common import PoolingParamsUpdate
|
||||
|
||||
|
||||
class Pooler(nn.Module, ABC):
|
||||
"""The interface required for all poolers used in pooling models in vLLM."""
|
||||
|
||||
@abstractmethod
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
"""Determine which pooling tasks are supported."""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
"""Construct the updated pooling parameters to use for a supported task."""
|
||||
return PoolingParamsUpdate()
|
||||
|
||||
@abstractmethod
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> PoolerOutput:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
__all__ = ["Pooler"]
|
||||
@@ -1,158 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable
|
||||
from typing import TypeVar
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_act_fn(
|
||||
config: PreTrainedConfig,
|
||||
static_num_labels: bool = True,
|
||||
) -> "PoolerActivation":
|
||||
# get classification act_fn
|
||||
# Implement alignment with transformers ForSequenceClassificationLoss
|
||||
# https://github.com/huggingface/transformers/blob/57bb6db6ee4cfaccc45b8d474dfad5a17811ca60/src/transformers/loss/loss_utils.py#L92
|
||||
num_labels: int | None = None
|
||||
if static_num_labels:
|
||||
num_labels = getattr(config, "num_labels", 0)
|
||||
|
||||
problem_type = getattr(config, "problem_type", "")
|
||||
if problem_type == "regression":
|
||||
return PoolerIdentity()
|
||||
if problem_type == "single_label_classification":
|
||||
return PoolerClassify(num_labels=num_labels)
|
||||
if problem_type == "multi_label_classification":
|
||||
return PoolerMultiLabelClassify()
|
||||
|
||||
# get cross_encoder act_fn
|
||||
function_name: str | None = None
|
||||
if (
|
||||
hasattr(config, "sentence_transformers")
|
||||
and "activation_fn" in config.sentence_transformers
|
||||
):
|
||||
function_name = config.sentence_transformers["activation_fn"]
|
||||
elif (
|
||||
hasattr(config, "sbert_ce_default_activation_function")
|
||||
and config.sbert_ce_default_activation_function is not None
|
||||
):
|
||||
function_name = config.sbert_ce_default_activation_function
|
||||
|
||||
if function_name is not None:
|
||||
if not function_name.startswith("torch.nn.modules."):
|
||||
raise ValueError(
|
||||
"Loading of activation functions is restricted to "
|
||||
"torch.nn.modules for security reasons"
|
||||
)
|
||||
fn = resolve_obj_by_qualname(function_name)()
|
||||
return PoolerActivation.wraps(fn)
|
||||
|
||||
return PoolerClassify(num_labels=num_labels)
|
||||
|
||||
|
||||
def resolve_classifier_act_fn(
|
||||
model_config: ModelConfig,
|
||||
static_num_labels: bool = True,
|
||||
act_fn: "PoolerActivation | None" = None,
|
||||
) -> "PoolerActivation":
|
||||
if act_fn is None:
|
||||
return get_act_fn(model_config.hf_config, static_num_labels)
|
||||
|
||||
if not callable(act_fn):
|
||||
raise TypeError(f"Expected a callable activation function, got {type(act_fn)}")
|
||||
return act_fn
|
||||
|
||||
|
||||
_T = TypeVar("_T", torch.Tensor, list[torch.Tensor])
|
||||
|
||||
|
||||
class PoolerActivation(nn.Module, ABC):
|
||||
@staticmethod
|
||||
def wraps(module: nn.Module) -> "PoolerActivation":
|
||||
if isinstance(module, nn.Identity):
|
||||
return PoolerIdentity()
|
||||
if isinstance(module, (nn.Sigmoid, nn.Softmax)):
|
||||
return PoolerClassify()
|
||||
|
||||
return LambdaPoolerActivation(module)
|
||||
|
||||
@abstractmethod
|
||||
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
def forward(self, pooled_data: _T) -> _T:
|
||||
# shape:
|
||||
# classify -> (batch_size, num_classes)
|
||||
# embed -> (batch_size, embedding_size) or list(embedding_size)
|
||||
if isinstance(pooled_data, list):
|
||||
return [self.forward_chunk(data) for data in pooled_data]
|
||||
|
||||
return self.forward_chunk(pooled_data)
|
||||
|
||||
|
||||
class PoolerIdentity(PoolerActivation):
|
||||
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
|
||||
return pooled_data
|
||||
|
||||
|
||||
class PoolerNormalize(PoolerActivation):
|
||||
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
|
||||
return F.normalize(pooled_data, p=2, dim=-1)
|
||||
|
||||
|
||||
class PoolerMultiLabelClassify(PoolerActivation):
|
||||
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
|
||||
return F.sigmoid(pooled_data)
|
||||
|
||||
|
||||
class PoolerClassify(PoolerActivation):
|
||||
def __init__(self, *, num_labels: int | None = None) -> None:
|
||||
super().__init__()
|
||||
|
||||
if num_labels == 0:
|
||||
logger.warning(
|
||||
"num_labels should be > 0 for classification "
|
||||
"models, falling back to sigmoid. "
|
||||
"Please check if the configuration is correct."
|
||||
)
|
||||
|
||||
self.num_labels = num_labels
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"num_labels={self.num_labels}"
|
||||
|
||||
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
|
||||
num_labels = self.num_labels
|
||||
if num_labels is None:
|
||||
num_labels = pooled_data.shape[-1]
|
||||
|
||||
if num_labels < 2:
|
||||
return F.sigmoid(pooled_data)
|
||||
|
||||
return F.softmax(pooled_data, dim=-1)
|
||||
|
||||
|
||||
class LambdaPoolerActivation(PoolerActivation):
|
||||
def __init__(self, fn: Callable[[torch.Tensor], torch.Tensor]) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.fn = fn
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
name = getattr(self.fn, "__name__", None)
|
||||
if name is None:
|
||||
name = self.fn.__class__.__name__
|
||||
return f"fn={name}"
|
||||
|
||||
def forward_chunk(self, pooled_data: torch.Tensor) -> torch.Tensor:
|
||||
return self.fn(pooled_data)
|
||||
@@ -1,32 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import TypeVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.pooling_params import PoolingParams
|
||||
|
||||
_T = TypeVar("_T", bound=torch.Tensor | list[torch.Tensor])
|
||||
|
||||
ProjectorFn = Callable[[torch.Tensor], torch.Tensor]
|
||||
ClassifierFn = Callable[[torch.Tensor], torch.Tensor]
|
||||
ActivationFn = Callable[[_T], _T]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PoolingParamsUpdate:
|
||||
requires_token_ids: bool = False
|
||||
"""Set this flag to enable CPU prompt token IDs for your pooler."""
|
||||
|
||||
def __or__(self, other: "PoolingParamsUpdate") -> "PoolingParamsUpdate":
|
||||
return PoolingParamsUpdate(
|
||||
requires_token_ids=self.requires_token_ids or other.requires_token_ids,
|
||||
)
|
||||
|
||||
def apply(self, params: PoolingParams) -> None:
|
||||
params.requires_token_ids = self.requires_token_ids
|
||||
|
||||
|
||||
__all__ = ["ActivationFn", "ClassifierFn", "ProjectorFn", "PoolingParamsUpdate"]
|
||||
@@ -1,45 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Poolers that produce an output aggregating all tokens in the sequence."""
|
||||
|
||||
from .heads import (
|
||||
ClassifierPoolerHead,
|
||||
EmbeddingPoolerHead,
|
||||
SequencePoolerHead,
|
||||
SequencePoolerHeadOutput,
|
||||
)
|
||||
from .methods import (
|
||||
CLSPool,
|
||||
LastPool,
|
||||
MeanPool,
|
||||
SequencePoolingMethod,
|
||||
SequencePoolingMethodOutput,
|
||||
get_seq_pooling_method,
|
||||
)
|
||||
from .poolers import (
|
||||
SequencePooler,
|
||||
SequencePoolerOutput,
|
||||
SequencePoolingFn,
|
||||
SequencePoolingHeadFn,
|
||||
pooler_for_classify,
|
||||
pooler_for_embed,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SequencePoolerHead",
|
||||
"SequencePoolerHeadOutput",
|
||||
"ClassifierPoolerHead",
|
||||
"EmbeddingPoolerHead",
|
||||
"SequencePoolingMethod",
|
||||
"SequencePoolingMethodOutput",
|
||||
"CLSPool",
|
||||
"LastPool",
|
||||
"MeanPool",
|
||||
"get_seq_pooling_method",
|
||||
"SequencePooler",
|
||||
"SequencePoolingFn",
|
||||
"SequencePoolingHeadFn",
|
||||
"SequencePoolerOutput",
|
||||
"pooler_for_classify",
|
||||
"pooler_for_embed",
|
||||
]
|
||||
@@ -1,204 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Set
|
||||
from typing import TypeAlias
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.layers.pooler import ActivationFn, ClassifierFn, ProjectorFn
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .methods import SequencePoolingMethodOutput
|
||||
|
||||
SequencePoolerHeadOutput: TypeAlias = torch.Tensor | list[torch.Tensor]
|
||||
|
||||
|
||||
class SequencePoolerHead(nn.Module, ABC):
|
||||
@abstractmethod
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def forward(
|
||||
self,
|
||||
pooled_data: SequencePoolingMethodOutput,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> SequencePoolerHeadOutput:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class EmbeddingPoolerHead(SequencePoolerHead):
|
||||
def __init__(
|
||||
self,
|
||||
projector: ProjectorFn | None = None,
|
||||
head_dtype: torch.dtype | str | None = None,
|
||||
activation: ActivationFn | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.projector = projector
|
||||
self.head_dtype = head_dtype
|
||||
self.activation = activation
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
attrs = []
|
||||
if self.head_dtype is not None:
|
||||
attrs.append(f"head_dtype={self.head_dtype}")
|
||||
if self.projector is not None:
|
||||
attrs.append("projector=True")
|
||||
if self.activation is not None:
|
||||
attrs.append(f"activation={self.activation.__class__.__name__}")
|
||||
return ", ".join(attrs)
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return {"embed"}
|
||||
|
||||
def forward(
|
||||
self,
|
||||
pooled_data: SequencePoolingMethodOutput,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> SequencePoolerHeadOutput:
|
||||
pooling_params = pooling_metadata.pooling_params
|
||||
if len(pooled_data) != len(pooling_params):
|
||||
raise ValueError(
|
||||
f"pooled_data length ({len(pooled_data)}) does not match "
|
||||
f"pooling_params length ({len(pooling_params)})"
|
||||
)
|
||||
|
||||
if isinstance(pooled_data, list):
|
||||
pooled_data = torch.stack(pooled_data)
|
||||
# pooled_data shape: [batchsize, hidden_size]
|
||||
|
||||
if self.head_dtype is not None:
|
||||
pooled_data = pooled_data.to(self.head_dtype)
|
||||
|
||||
# Apply ST projector
|
||||
if self.projector is not None:
|
||||
embeddings = self.projector(pooled_data)
|
||||
else:
|
||||
embeddings = pooled_data
|
||||
# embeddings shape: [batchsize, embedding_size]
|
||||
|
||||
# for matryoshka representation
|
||||
dimensions_list = [pooling_param.dimensions for pooling_param in pooling_params]
|
||||
if any(d is not None for d in dimensions_list):
|
||||
# change the output dimension
|
||||
if len(embeddings) != len(dimensions_list):
|
||||
raise ValueError(
|
||||
f"embeddings length ({len(embeddings)}) does not match "
|
||||
f"dimensions_list length ({len(dimensions_list)})"
|
||||
)
|
||||
if len(set(dimensions_list)) == 1 and not isinstance(embeddings, list):
|
||||
# if all dimensions are the same
|
||||
d = dimensions_list[0]
|
||||
embeddings = embeddings[..., :d]
|
||||
else:
|
||||
embeddings = [
|
||||
vecs if d is None else vecs[..., :d]
|
||||
for vecs, d in zip(embeddings, dimensions_list)
|
||||
]
|
||||
|
||||
# for normalize
|
||||
if self.activation is not None:
|
||||
flags = [p.use_activation for p in pooling_params]
|
||||
if len(set(flags)) == 1:
|
||||
if flags[0]:
|
||||
embeddings = self.activation(embeddings)
|
||||
else:
|
||||
embeddings = [
|
||||
self.activation(vecs) if f else vecs
|
||||
for vecs, f in zip(embeddings, flags)
|
||||
]
|
||||
|
||||
# embeddings shape: [batchsize, embedding_size]
|
||||
return embeddings
|
||||
|
||||
|
||||
class ClassifierPoolerHead(SequencePoolerHead):
|
||||
def __init__(
|
||||
self,
|
||||
classifier: ClassifierFn | None = None,
|
||||
logit_mean: float | None = None,
|
||||
logit_sigma: float | None = None,
|
||||
head_dtype: torch.dtype | str | None = None,
|
||||
activation: ActivationFn | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.classifier = classifier
|
||||
self.logit_mean = logit_mean
|
||||
self.logit_sigma = logit_sigma
|
||||
self.head_dtype = head_dtype
|
||||
self.activation = activation
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
attrs = []
|
||||
if self.head_dtype is not None:
|
||||
attrs.append(f"head_dtype={self.head_dtype}")
|
||||
if self.classifier is not None:
|
||||
attrs.append("classifier=True")
|
||||
if self.logit_mean is not None:
|
||||
attrs.append(f"logit_mean={self.logit_mean}")
|
||||
if self.logit_sigma is not None:
|
||||
attrs.append(f"logit_sigma={self.logit_sigma}")
|
||||
if self.activation is not None:
|
||||
attrs.append(f"activation={self.activation.__class__.__name__}")
|
||||
return ", ".join(attrs)
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return {"classify"}
|
||||
|
||||
def forward(
|
||||
self,
|
||||
pooled_data: SequencePoolingMethodOutput,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> SequencePoolerHeadOutput:
|
||||
pooling_params = pooling_metadata.pooling_params
|
||||
if len(pooled_data) != len(pooling_params):
|
||||
raise ValueError(
|
||||
f"pooled_data length ({len(pooled_data)}) does not match "
|
||||
f"pooling_params length ({len(pooling_params)})"
|
||||
)
|
||||
|
||||
if isinstance(pooled_data, list):
|
||||
pooled_data = torch.stack(pooled_data)
|
||||
# pooled_data shape: [batchsize, hidden_size]
|
||||
|
||||
if self.head_dtype is not None:
|
||||
pooled_data = pooled_data.to(self.head_dtype)
|
||||
|
||||
if self.classifier is not None:
|
||||
logits: SequencePoolerHeadOutput = self.classifier(pooled_data)
|
||||
else:
|
||||
logits = pooled_data
|
||||
|
||||
# logits shape: [batchsize, num_labels]
|
||||
# Affine score calibration: activation((logit - mean) / sigma)
|
||||
if self.logit_mean is not None:
|
||||
logits = (
|
||||
[value - self.logit_mean for value in logits]
|
||||
if isinstance(logits, list)
|
||||
else logits - self.logit_mean
|
||||
)
|
||||
if self.logit_sigma is not None:
|
||||
logits = (
|
||||
[value / self.logit_sigma for value in logits]
|
||||
if isinstance(logits, list)
|
||||
else logits / self.logit_sigma
|
||||
)
|
||||
|
||||
if self.activation is not None:
|
||||
flags = [p.use_activation for p in pooling_params]
|
||||
if len(set(flags)) == 1:
|
||||
logits = self.activation(logits) if flags[0] else logits
|
||||
else:
|
||||
logits = [
|
||||
self.activation(vecs) if f else vecs
|
||||
for vecs, f in zip(logits, flags)
|
||||
]
|
||||
|
||||
# logits shape: [batchsize, num_labels]
|
||||
return logits
|
||||
@@ -1,119 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Set
|
||||
from typing import TypeAlias
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config.pooler import SequencePoolingType
|
||||
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
SequencePoolingMethodOutput: TypeAlias = torch.Tensor | list[torch.Tensor]
|
||||
|
||||
_MEAN_POOL_ACCUMULATION_CHUNK_BYTES = 16 * 1024 * 1024 # 16MB
|
||||
|
||||
|
||||
class SequencePoolingMethod(nn.Module, ABC):
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return {"token_embed", "token_classify", "embed", "classify"}
|
||||
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
return PoolingParamsUpdate()
|
||||
|
||||
@abstractmethod
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> SequencePoolingMethodOutput:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class CLSPool(SequencePoolingMethod):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> SequencePoolingMethodOutput:
|
||||
pooling_cursor = pooling_metadata.get_pooling_cursor()
|
||||
if pooling_cursor.is_partial_prefill():
|
||||
raise RuntimeError("partial prefill is not supported with CLS pooling")
|
||||
|
||||
return hidden_states[pooling_cursor.first_token_indices_gpu]
|
||||
|
||||
|
||||
class LastPool(SequencePoolingMethod):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> SequencePoolingMethodOutput:
|
||||
pooling_cursor = pooling_metadata.get_pooling_cursor()
|
||||
return hidden_states[pooling_cursor.last_token_indices_gpu]
|
||||
|
||||
|
||||
class MeanPool(SequencePoolingMethod):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> SequencePoolingMethodOutput:
|
||||
pooling_cursor = pooling_metadata.get_pooling_cursor()
|
||||
if pooling_cursor.is_partial_prefill():
|
||||
raise RuntimeError("partial prefill is not supported with MEAN pooling")
|
||||
|
||||
prompt_lens_cpu = pooling_cursor.prompt_lens_cpu
|
||||
num_seqs = prompt_lens_cpu.numel()
|
||||
hidden_size = hidden_states.shape[-1]
|
||||
|
||||
if num_seqs == 0:
|
||||
# early return for empty batch
|
||||
return hidden_states.new_empty((0, hidden_size), dtype=torch.float32)
|
||||
|
||||
prompt_lens = async_tensor_h2d(
|
||||
prompt_lens_cpu, device=hidden_states.device, dtype=torch.int64
|
||||
)
|
||||
# eg. [2, 1, 3] -> [0, 0, 1, 2, 2, 2]
|
||||
segment_ids = torch.repeat_interleave(
|
||||
torch.arange(num_seqs, device=hidden_states.device, dtype=torch.long),
|
||||
prompt_lens,
|
||||
output_size=int(prompt_lens_cpu.sum()),
|
||||
)
|
||||
segment_sums = torch.zeros(
|
||||
(num_seqs, hidden_size),
|
||||
dtype=torch.float32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
bytes_per_token = hidden_size * torch.finfo(torch.float32).bits // 8
|
||||
chunk_size = max(1, _MEAN_POOL_ACCUMULATION_CHUNK_BYTES // bytes_per_token)
|
||||
|
||||
# iterate over the batch in chunks
|
||||
for start in range(0, hidden_states.shape[0], chunk_size):
|
||||
end = min(start + chunk_size, hidden_states.shape[0])
|
||||
# using index_add_ to accumulate for each segment
|
||||
segment_sums.index_add_(
|
||||
0,
|
||||
segment_ids[start:end],
|
||||
hidden_states[start:end].to(dtype=torch.float32),
|
||||
)
|
||||
|
||||
return segment_sums / prompt_lens.unsqueeze(1)
|
||||
|
||||
|
||||
def get_seq_pooling_method(
|
||||
pooling_type: SequencePoolingType | str,
|
||||
) -> SequencePoolingMethod:
|
||||
if pooling_type == "CLS":
|
||||
return CLSPool()
|
||||
if pooling_type == "LAST":
|
||||
return LastPool()
|
||||
if pooling_type == "MEAN":
|
||||
return MeanPool()
|
||||
|
||||
raise NotImplementedError(f"Unknown sequence pooling type: {pooling_type!r}")
|
||||
@@ -1,137 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable, Set
|
||||
from typing import TypeAlias
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import PoolerConfig, get_current_vllm_config
|
||||
from vllm.model_executor.layers.pooler import ClassifierFn, PoolingParamsUpdate
|
||||
from vllm.model_executor.layers.pooler.abstract import Pooler
|
||||
from vllm.model_executor.layers.pooler.activations import (
|
||||
PoolerActivation,
|
||||
PoolerNormalize,
|
||||
resolve_classifier_act_fn,
|
||||
)
|
||||
from vllm.model_executor.models.adapters import _load_st_projector
|
||||
from vllm.tasks import POOLING_TASKS, PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .heads import (
|
||||
ClassifierPoolerHead,
|
||||
EmbeddingPoolerHead,
|
||||
SequencePoolerHead,
|
||||
SequencePoolerHeadOutput,
|
||||
)
|
||||
from .methods import (
|
||||
SequencePoolingMethod,
|
||||
SequencePoolingMethodOutput,
|
||||
get_seq_pooling_method,
|
||||
)
|
||||
|
||||
SequencePoolingFn: TypeAlias = Callable[
|
||||
[torch.Tensor, PoolingMetadata],
|
||||
SequencePoolingMethodOutput,
|
||||
]
|
||||
SequencePoolingHeadFn: TypeAlias = Callable[
|
||||
[SequencePoolingMethodOutput, PoolingMetadata],
|
||||
SequencePoolerHeadOutput,
|
||||
]
|
||||
|
||||
SequencePoolerOutput: TypeAlias = torch.Tensor | list[torch.Tensor]
|
||||
|
||||
|
||||
class SequencePooler(Pooler):
|
||||
"""A layer that pools specific information from hidden states.
|
||||
|
||||
This layer does the following:
|
||||
1. Extracts specific tokens or aggregates data based on pooling method.
|
||||
2. Postprocesses the output based on pooling head.
|
||||
3. Returns structured results as `PoolerOutput`.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pooling: SequencePoolingMethod | SequencePoolingFn,
|
||||
head: SequencePoolerHead | SequencePoolingHeadFn,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.pooling = pooling
|
||||
self.head = head
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return (
|
||||
f"pooling={self.pooling.__class__.__name__}, "
|
||||
f"head={self.head.__class__.__name__}"
|
||||
)
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
tasks = set(POOLING_TASKS)
|
||||
|
||||
if isinstance(self.pooling, SequencePoolingMethod):
|
||||
tasks &= self.pooling.get_supported_tasks()
|
||||
if isinstance(self.head, SequencePoolerHead):
|
||||
tasks &= self.head.get_supported_tasks()
|
||||
|
||||
return tasks
|
||||
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
updates = PoolingParamsUpdate()
|
||||
|
||||
if isinstance(self.pooling, SequencePoolingMethod):
|
||||
updates |= self.pooling.get_pooling_updates(task)
|
||||
|
||||
return updates
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> SequencePoolerOutput:
|
||||
pooled_data = self.pooling(hidden_states, pooling_metadata)
|
||||
pooled_data = self.head(pooled_data, pooling_metadata)
|
||||
return pooled_data
|
||||
|
||||
|
||||
def pooler_for_embed(pooler_config: PoolerConfig) -> SequencePooler:
|
||||
pooling = get_seq_pooling_method(pooler_config.get_seq_pooling_type())
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
model_config = vllm_config.model_config
|
||||
head = EmbeddingPoolerHead(
|
||||
head_dtype=model_config.head_dtype,
|
||||
projector=_load_st_projector(model_config),
|
||||
activation=PoolerNormalize(),
|
||||
)
|
||||
|
||||
return SequencePooler(pooling=pooling, head=head)
|
||||
|
||||
|
||||
def pooler_for_classify(
|
||||
pooler_config: PoolerConfig,
|
||||
*,
|
||||
pooling: SequencePoolingMethod | SequencePoolingFn | None = None,
|
||||
classifier: ClassifierFn | None = None,
|
||||
act_fn: PoolerActivation | None = None,
|
||||
) -> SequencePooler:
|
||||
if pooling is None:
|
||||
pooling = get_seq_pooling_method(pooler_config.get_seq_pooling_type())
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
model_config = vllm_config.model_config
|
||||
if model_config.pooler_config is None:
|
||||
raise ValueError(
|
||||
"model_config.pooler_config must be set for classification pooling"
|
||||
)
|
||||
head = ClassifierPoolerHead(
|
||||
head_dtype=model_config.head_dtype,
|
||||
classifier=classifier,
|
||||
logit_mean=model_config.pooler_config.logit_mean,
|
||||
logit_sigma=model_config.pooler_config.logit_sigma,
|
||||
activation=resolve_classifier_act_fn(
|
||||
model_config, static_num_labels=False, act_fn=act_fn
|
||||
),
|
||||
)
|
||||
|
||||
return SequencePooler(pooling=pooling, head=head)
|
||||
@@ -1,258 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import dataclasses
|
||||
from collections.abc import Mapping, Set
|
||||
from itertools import groupby
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import PoolerConfig
|
||||
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .abstract import Pooler, PoolerOutput
|
||||
from .common import ClassifierFn
|
||||
from .seqwise import (
|
||||
SequencePoolingFn,
|
||||
SequencePoolingMethod,
|
||||
pooler_for_classify,
|
||||
pooler_for_embed,
|
||||
)
|
||||
from .tokwise import pooler_for_token_classify, pooler_for_token_embed
|
||||
|
||||
|
||||
class DispatchPooler(Pooler):
|
||||
"""Dispatches calls to a sub-pooler based on the pooling task."""
|
||||
|
||||
@classmethod
|
||||
def for_embedding(cls, pooler_config: PoolerConfig):
|
||||
return cls(
|
||||
{
|
||||
"token_embed": pooler_for_token_embed(pooler_config),
|
||||
"embed": pooler_for_embed(pooler_config),
|
||||
},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def for_seq_cls(
|
||||
cls,
|
||||
pooler_config: PoolerConfig,
|
||||
*,
|
||||
pooling: SequencePoolingMethod | SequencePoolingFn | None = None,
|
||||
classifier: ClassifierFn | None = None,
|
||||
):
|
||||
return cls(
|
||||
{
|
||||
"token_classify": pooler_for_token_classify(
|
||||
pooler_config,
|
||||
classifier=classifier,
|
||||
),
|
||||
"classify": pooler_for_classify(
|
||||
pooler_config,
|
||||
pooling=pooling,
|
||||
classifier=classifier,
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, poolers_by_task: Mapping[PoolingTask, Pooler]) -> None:
|
||||
super().__init__()
|
||||
|
||||
for task, pooler in poolers_by_task.items():
|
||||
if task not in pooler.get_supported_tasks():
|
||||
raise ValueError(
|
||||
f"{pooler=} does not support {task=}. "
|
||||
f"Supported tasks: {pooler.get_supported_tasks()}"
|
||||
)
|
||||
|
||||
self.poolers_by_task = poolers_by_task
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return set(self.poolers_by_task)
|
||||
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
return self.poolers_by_task[task].get_pooling_updates(task)
|
||||
|
||||
def replace_classifier(
|
||||
self,
|
||||
new_classifier: ClassifierFn,
|
||||
) -> None:
|
||||
"""Replaces the classifier to the LoRA-wrapped version."""
|
||||
for task, pooler in self.poolers_by_task.items():
|
||||
if task not in {"classify"}:
|
||||
# Now LoRA only supports classification tasks,
|
||||
# so we only replace the classifier for "classify" task.
|
||||
continue
|
||||
head = getattr(pooler, "head", None)
|
||||
if head is not None and hasattr(head, "classifier"):
|
||||
head.classifier = new_classifier
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> PoolerOutput:
|
||||
poolers_by_task = self.poolers_by_task
|
||||
cursor = pooling_metadata.pooling_cursor
|
||||
|
||||
outputs = list[torch.Tensor | None]()
|
||||
offset = 0
|
||||
token_offset = 0
|
||||
for task, group in groupby(pooling_metadata.tasks):
|
||||
if not (pooler := poolers_by_task.get(task)):
|
||||
raise ValueError(
|
||||
f"Unsupported task: {task!r} "
|
||||
f"Supported tasks: {self.get_supported_tasks()}"
|
||||
)
|
||||
|
||||
num_items = len(list(group))
|
||||
group_metadata = pooling_metadata[offset : offset + num_items]
|
||||
if cursor is None:
|
||||
group_hidden_states = hidden_states
|
||||
else:
|
||||
# Slice out this group's tokens so sub-poolers see only their
|
||||
# portion of the batch. Token offset is computed from the CPU
|
||||
# `num_scheduled_tokens_cpu` to avoid a GPU->CPU sync.
|
||||
group_cursor = group_metadata.pooling_cursor
|
||||
assert group_cursor is not None
|
||||
num_group_tokens = int(group_cursor.num_scheduled_tokens_cpu.sum())
|
||||
group_hidden_states = hidden_states[
|
||||
token_offset : token_offset + num_group_tokens
|
||||
]
|
||||
if token_offset:
|
||||
# Shift first/last indices to be relative to the slice
|
||||
# so seqwise poolers (which index `hidden_states` directly)
|
||||
# remain correct.
|
||||
pooling_cursor = dataclasses.replace(
|
||||
group_cursor,
|
||||
first_token_indices_gpu=(
|
||||
group_cursor.first_token_indices_gpu - token_offset
|
||||
),
|
||||
last_token_indices_gpu=(
|
||||
group_cursor.last_token_indices_gpu - token_offset
|
||||
),
|
||||
)
|
||||
group_metadata = dataclasses.replace(
|
||||
group_metadata, pooling_cursor=pooling_cursor
|
||||
)
|
||||
token_offset += num_group_tokens
|
||||
|
||||
group_output: PoolerOutput = pooler(group_hidden_states, group_metadata)
|
||||
|
||||
outputs.extend(group_output)
|
||||
offset += num_items
|
||||
|
||||
return outputs
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
s = f"supported_task={self.get_supported_tasks()}"
|
||||
return s
|
||||
|
||||
|
||||
class IdentityPooler(Pooler):
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return {"plugin"}
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> PoolerOutput:
|
||||
return hidden_states
|
||||
|
||||
|
||||
class BOSEOSFilter(Pooler):
|
||||
"""Filters the BOS and EOS token results from outputs."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pooler: Pooler,
|
||||
bos_token_id: int = -1, # -1 disables the filtering
|
||||
eos_token_id: int = -1,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.pooler = pooler
|
||||
self.bos_token_id = bos_token_id
|
||||
self.eos_token_id = eos_token_id
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"bos_token_id={self.bos_token_id}, eos_token_id={self.eos_token_id}"
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return self.pooler.get_supported_tasks()
|
||||
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
return PoolingParamsUpdate(requires_token_ids=True)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor | list[torch.Tensor],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> PoolerOutput:
|
||||
pooled_outputs = self.pooler(hidden_states, pooling_metadata)
|
||||
assert isinstance(pooled_outputs, list)
|
||||
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
|
||||
|
||||
for i, (prompt_len, token_ids) in enumerate(
|
||||
zip(pooling_metadata.prompt_lens, prompt_token_ids)
|
||||
):
|
||||
# Late chunking returns one row per chunk, with special tokens
|
||||
# already included in the means. Skip BOS/EOS row removal for these
|
||||
# requests; ordinary token outputs still use the filtering below.
|
||||
if pooling_metadata.pooling_params[i].late_chunking_params is not None:
|
||||
continue
|
||||
pooled_data = pooled_outputs[i]
|
||||
assert (
|
||||
isinstance(pooled_data, torch.Tensor)
|
||||
and pooled_data.shape[0] == prompt_len
|
||||
)
|
||||
if int(token_ids[0]) == self.bos_token_id:
|
||||
pooled_data = pooled_data[1:]
|
||||
if int(token_ids[-1]) == self.eos_token_id:
|
||||
pooled_data = pooled_data[:-1]
|
||||
pooled_outputs[i] = pooled_data.squeeze(-1)
|
||||
|
||||
return pooled_outputs
|
||||
|
||||
|
||||
class BgeM3Pooler(Pooler):
|
||||
def __init__(self, token_classify_pooler: Pooler, embed_pooler: Pooler) -> None:
|
||||
super().__init__()
|
||||
self.token_classify_pooler = token_classify_pooler
|
||||
self.embed_pooler = embed_pooler
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, pooling_metadata: PoolingMetadata
|
||||
) -> PoolerOutput:
|
||||
embed_outputs = self.embed_pooler(hidden_states, pooling_metadata)
|
||||
token_classify_outputs = self.token_classify_pooler(
|
||||
hidden_states, pooling_metadata
|
||||
)
|
||||
pooler_outputs: list[torch.Tensor] = []
|
||||
for embed_output, token_classify_output in zip(
|
||||
embed_outputs, token_classify_outputs
|
||||
):
|
||||
pooler_outputs.append(
|
||||
torch.cat(
|
||||
[embed_output.view(-1), token_classify_output.view(-1)], dim=-1
|
||||
)
|
||||
)
|
||||
|
||||
return pooler_outputs
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return {"embed&token_classify"}
|
||||
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
return self.embed_pooler.get_pooling_updates(
|
||||
"embed"
|
||||
) | self.token_classify_pooler.get_pooling_updates("token_classify")
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
s = f"supported_task={self.get_supported_tasks()}"
|
||||
return s
|
||||
|
||||
|
||||
__all__ = ["BOSEOSFilter", "DispatchPooler", "IdentityPooler", "BgeM3Pooler"]
|
||||
@@ -1,43 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Poolers that produce an output for each token in the sequence."""
|
||||
|
||||
from .heads import (
|
||||
TokenClassifierPoolerHead,
|
||||
TokenEmbeddingPoolerHead,
|
||||
TokenPoolerHead,
|
||||
TokenPoolerHeadOutputItem,
|
||||
)
|
||||
from .methods import (
|
||||
AllPool,
|
||||
StepPool,
|
||||
TokenPoolingMethod,
|
||||
TokenPoolingMethodOutputItem,
|
||||
get_tok_pooling_method,
|
||||
)
|
||||
from .poolers import (
|
||||
TokenPooler,
|
||||
TokenPoolerOutput,
|
||||
TokenPoolingFn,
|
||||
TokenPoolingHeadFn,
|
||||
pooler_for_token_classify,
|
||||
pooler_for_token_embed,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"TokenPoolerHead",
|
||||
"TokenPoolerHeadOutputItem",
|
||||
"TokenClassifierPoolerHead",
|
||||
"TokenEmbeddingPoolerHead",
|
||||
"TokenPoolingMethod",
|
||||
"TokenPoolingMethodOutputItem",
|
||||
"AllPool",
|
||||
"StepPool",
|
||||
"get_tok_pooling_method",
|
||||
"TokenPooler",
|
||||
"TokenPoolerOutput",
|
||||
"TokenPoolingFn",
|
||||
"TokenPoolingHeadFn",
|
||||
"pooler_for_token_classify",
|
||||
"pooler_for_token_embed",
|
||||
]
|
||||
@@ -1,192 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Set
|
||||
from typing import TypeAlias
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.layers.pooler import ActivationFn, ClassifierFn, ProjectorFn
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .methods import TokenPoolingMethodOutputItem
|
||||
|
||||
TokenPoolerHeadOutputItem: TypeAlias = torch.Tensor | None
|
||||
|
||||
|
||||
def _mean_pool_chunks(data: torch.Tensor, chunk_size: int) -> torch.Tensor:
|
||||
"""Mean-pool complete contextual states, accumulating in FP32."""
|
||||
chunk_size = min(chunk_size, len(data))
|
||||
num_full, remainder = divmod(len(data), chunk_size)
|
||||
full_end = num_full * chunk_size
|
||||
full_chunks = data[:full_end].reshape(num_full, chunk_size, data.shape[-1])
|
||||
if not remainder:
|
||||
return full_chunks.mean(dim=1, dtype=torch.float32)
|
||||
|
||||
return torch.cat(
|
||||
[
|
||||
full_chunks.mean(dim=1, dtype=torch.float32),
|
||||
data[full_end:].mean(dim=0, keepdim=True, dtype=torch.float32),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class TokenPoolerHead(nn.Module, ABC):
|
||||
@abstractmethod
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def forward_chunk(
|
||||
self,
|
||||
pooled_data: TokenPoolingMethodOutputItem,
|
||||
pooling_param: PoolingParams,
|
||||
) -> TokenPoolerHeadOutputItem:
|
||||
raise NotImplementedError
|
||||
|
||||
def forward(
|
||||
self,
|
||||
pooled_data: list[TokenPoolingMethodOutputItem],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> list[TokenPoolerHeadOutputItem]:
|
||||
pooling_params = pooling_metadata.pooling_params
|
||||
if len(pooled_data) != len(pooling_params):
|
||||
raise ValueError(
|
||||
f"pooled_data length ({len(pooled_data)}) does not match "
|
||||
f"pooling_params length ({len(pooling_params)})"
|
||||
)
|
||||
|
||||
return [self.forward_chunk(d, p) for d, p in zip(pooled_data, pooling_params)]
|
||||
|
||||
|
||||
class TokenEmbeddingPoolerHead(TokenPoolerHead):
|
||||
def __init__(
|
||||
self,
|
||||
head_dtype: torch.dtype | str | None = None,
|
||||
projector: ProjectorFn | None = None,
|
||||
activation: ActivationFn | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.head_dtype = head_dtype
|
||||
self.projector = projector
|
||||
self.activation = activation
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
attrs = []
|
||||
if self.head_dtype is not None:
|
||||
attrs.append(f"head_dtype={self.head_dtype}")
|
||||
if self.projector is not None:
|
||||
attrs.append("projector=True")
|
||||
if self.activation is not None:
|
||||
attrs.append(f"activation={self.activation.__class__.__name__}")
|
||||
return ", ".join(attrs)
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return {"token_embed"}
|
||||
|
||||
def forward_chunk(
|
||||
self,
|
||||
pooled_data: TokenPoolingMethodOutputItem,
|
||||
pooling_param: PoolingParams,
|
||||
) -> TokenPoolerHeadOutputItem:
|
||||
# for unfinished chunked prefill
|
||||
if pooled_data is None:
|
||||
return None
|
||||
|
||||
if (late_chunking := pooling_param.late_chunking_params) is not None:
|
||||
means = _mean_pool_chunks(pooled_data, late_chunking.chunk_size)
|
||||
pooled_data = (
|
||||
means.to(pooled_data.dtype) if self.head_dtype is None else means
|
||||
)
|
||||
|
||||
if self.head_dtype is not None:
|
||||
pooled_data = pooled_data.to(self.head_dtype)
|
||||
# pooled_data shape: [n_tokens, hidden_size]
|
||||
|
||||
# Apply ST projector
|
||||
if self.projector is not None:
|
||||
embeddings = self.projector(pooled_data)
|
||||
else:
|
||||
embeddings = pooled_data
|
||||
# embeddings shape: [n_tokens, embedding_size]
|
||||
|
||||
# for matryoshka representation
|
||||
if pooling_param.dimensions is not None:
|
||||
embeddings = embeddings[..., : pooling_param.dimensions]
|
||||
|
||||
# for normalize
|
||||
if self.activation is not None and pooling_param.use_activation:
|
||||
embeddings = self.activation(embeddings)
|
||||
|
||||
# embeddings shape: [n_tokens, embedding_size]
|
||||
return embeddings
|
||||
|
||||
|
||||
class TokenClassifierPoolerHead(TokenPoolerHead):
|
||||
def __init__(
|
||||
self,
|
||||
classifier: ClassifierFn | None = None,
|
||||
logit_mean: float | None = None,
|
||||
logit_sigma: float | None = None,
|
||||
head_dtype: torch.dtype | str | None = None,
|
||||
activation: ActivationFn | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.classifier = classifier
|
||||
self.logit_mean = logit_mean
|
||||
self.logit_sigma = logit_sigma
|
||||
self.head_dtype = head_dtype
|
||||
self.activation = activation
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
attrs = []
|
||||
if self.head_dtype is not None:
|
||||
attrs.append(f"head_dtype={self.head_dtype}")
|
||||
if self.classifier is not None:
|
||||
attrs.append("classifier=True")
|
||||
if self.logit_mean is not None:
|
||||
attrs.append(f"logit_mean={self.logit_mean}")
|
||||
if self.logit_sigma is not None:
|
||||
attrs.append(f"logit_sigma={self.logit_sigma}")
|
||||
if self.activation is not None:
|
||||
attrs.append(f"activation={self.activation.__class__.__name__}")
|
||||
return ", ".join(attrs)
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return {"token_classify"}
|
||||
|
||||
def forward_chunk(
|
||||
self,
|
||||
pooled_data: TokenPoolingMethodOutputItem,
|
||||
pooling_param: PoolingParams,
|
||||
) -> TokenPoolerHeadOutputItem:
|
||||
# for unfinished chunked prefill
|
||||
if pooled_data is None:
|
||||
return None
|
||||
|
||||
if self.head_dtype is not None:
|
||||
pooled_data = pooled_data.to(self.head_dtype)
|
||||
# hidden_states shape: [n_token, hidden_size]
|
||||
|
||||
if self.classifier is not None:
|
||||
logits = self.classifier(pooled_data)
|
||||
else:
|
||||
logits = pooled_data
|
||||
# logits shape: [n_token, num_labels]
|
||||
|
||||
# Affine score calibration: activation((logit - mean) / sigma)
|
||||
if self.logit_mean is not None:
|
||||
logits = logits - self.logit_mean
|
||||
if self.logit_sigma is not None:
|
||||
logits = logits / self.logit_sigma
|
||||
|
||||
if self.activation is not None and pooling_param.use_activation:
|
||||
logits = self.activation(logits)
|
||||
|
||||
# logits shape: [n_token, num_labels]
|
||||
return logits
|
||||
@@ -1,149 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Set
|
||||
from typing import TypeAlias
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config.pooler import TokenPoolingType
|
||||
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
TokenPoolingMethodOutputItem: TypeAlias = torch.Tensor | None
|
||||
|
||||
|
||||
class TokenPoolingMethod(nn.Module, ABC):
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return {"token_embed", "token_classify"}
|
||||
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
return PoolingParamsUpdate()
|
||||
|
||||
@abstractmethod
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> list[TokenPoolingMethodOutputItem]:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class AllPool(TokenPoolingMethod):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
|
||||
self.enable_chunked_prefill = scheduler_config.enable_chunked_prefill
|
||||
# With async scheduling, the next step may overwrite the hidden states
|
||||
# buffer before outputs are copied to CPU, so they must own their storage.
|
||||
self.clone_finished = bool(scheduler_config.async_scheduling)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return (
|
||||
f"enable_chunked_prefill={self.enable_chunked_prefill}, "
|
||||
f"clone_finished={self.clone_finished}"
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> list[TokenPoolingMethodOutputItem]:
|
||||
pooling_cursor = pooling_metadata.get_pooling_cursor()
|
||||
# Use the already-CPU num_scheduled_tokens tensor so `.tolist()`
|
||||
# doesn't trigger a GPU->CPU sync. torch.split produces the same
|
||||
# consecutive slices as indexing with first/last per-sequence indices.
|
||||
hidden_states_lst = list(
|
||||
torch.split(hidden_states, pooling_cursor.num_scheduled_tokens_cpu.tolist())
|
||||
)
|
||||
|
||||
if not self.enable_chunked_prefill:
|
||||
if self.clone_finished:
|
||||
return [hs.clone() for hs in hidden_states_lst]
|
||||
return hidden_states_lst
|
||||
|
||||
pooling_states = pooling_metadata.pooling_states
|
||||
finished_mask = pooling_cursor.get_finished_mask()
|
||||
|
||||
# If chunked_prefill is enabled
|
||||
# 1. first store the chunked hidden_states in pooling_states.hidden_states_cache
|
||||
for p, hs_chunk, finished in zip(
|
||||
pooling_states, hidden_states_lst, finished_mask
|
||||
):
|
||||
# Own data retained across steps or returned without concatenation.
|
||||
needs_owned_storage = not finished or (
|
||||
self.clone_finished and not p.hidden_states_cache
|
||||
)
|
||||
p.hidden_states_cache.append(
|
||||
hs_chunk.clone() if needs_owned_storage else hs_chunk
|
||||
)
|
||||
|
||||
# 2. Once prefill is finished, send hidden_states_cache to PoolerHead
|
||||
output_list = list[TokenPoolingMethodOutputItem]()
|
||||
for p, finished in zip(pooling_states, finished_mask):
|
||||
if finished:
|
||||
hidden_states_cache = p.hidden_states_cache
|
||||
if len(hidden_states_cache) == 1:
|
||||
output_list.append(hidden_states_cache[0])
|
||||
else:
|
||||
output_list.append(torch.concat(hidden_states_cache, dim=0))
|
||||
p.clean()
|
||||
else:
|
||||
output_list.append(None)
|
||||
|
||||
return output_list
|
||||
|
||||
|
||||
class StepPool(AllPool):
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
return PoolingParamsUpdate(requires_token_ids=True)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> list[TokenPoolingMethodOutputItem]:
|
||||
pooled_data_lst = super().forward(hidden_states, pooling_metadata)
|
||||
# Use the CPU copy of prompt_token_ids so the step_tag_id mask can be
|
||||
# resolved to indices without a d2h sync from boolean indexing.
|
||||
prompt_token_ids_cpu = pooling_metadata.get_prompt_token_ids_cpu()
|
||||
pooling_params = pooling_metadata.pooling_params
|
||||
|
||||
pooled_data = list[torch.Tensor | None]()
|
||||
for data, token_id_cpu, pooling_param in zip(
|
||||
pooled_data_lst, prompt_token_ids_cpu, pooling_params
|
||||
):
|
||||
# for unfinished chunked prefill
|
||||
if data is None:
|
||||
pooled_data.append(None)
|
||||
else:
|
||||
step_tag_id = pooling_param.step_tag_id
|
||||
returned_token_ids = pooling_param.returned_token_ids
|
||||
|
||||
if returned_token_ids is not None and len(returned_token_ids) > 0:
|
||||
data = data[:, returned_token_ids]
|
||||
|
||||
if step_tag_id is not None:
|
||||
idx_cpu = (token_id_cpu == step_tag_id).nonzero(as_tuple=True)[0]
|
||||
idx = async_tensor_h2d(idx_cpu, data.device)
|
||||
data = data[idx]
|
||||
|
||||
pooled_data.append(data)
|
||||
|
||||
return pooled_data
|
||||
|
||||
|
||||
def get_tok_pooling_method(pooling_type: TokenPoolingType | str):
|
||||
if pooling_type == "ALL":
|
||||
return AllPool()
|
||||
if pooling_type == "STEP":
|
||||
return StepPool()
|
||||
|
||||
raise NotImplementedError(f"Unknown tokenwise pooling type: {pooling_type!r}")
|
||||
@@ -1,144 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable, Set
|
||||
from typing import TypeAlias
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import PoolerConfig, get_current_vllm_config
|
||||
from vllm.model_executor.layers.pooler import (
|
||||
ClassifierFn,
|
||||
PoolingParamsUpdate,
|
||||
ProjectorFn,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.abstract import Pooler
|
||||
from vllm.model_executor.layers.pooler.activations import (
|
||||
PoolerActivation,
|
||||
PoolerNormalize,
|
||||
resolve_classifier_act_fn,
|
||||
)
|
||||
from vllm.model_executor.models.adapters import _load_st_projector
|
||||
from vllm.tasks import POOLING_TASKS, PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .heads import (
|
||||
TokenClassifierPoolerHead,
|
||||
TokenEmbeddingPoolerHead,
|
||||
TokenPoolerHead,
|
||||
TokenPoolerHeadOutputItem,
|
||||
)
|
||||
from .methods import (
|
||||
TokenPoolingMethod,
|
||||
TokenPoolingMethodOutputItem,
|
||||
get_tok_pooling_method,
|
||||
)
|
||||
|
||||
TokenPoolingFn: TypeAlias = Callable[
|
||||
[torch.Tensor, PoolingMetadata],
|
||||
list[TokenPoolingMethodOutputItem],
|
||||
]
|
||||
TokenPoolingHeadFn: TypeAlias = Callable[
|
||||
[list[TokenPoolingMethodOutputItem], PoolingMetadata],
|
||||
list[TokenPoolerHeadOutputItem],
|
||||
]
|
||||
|
||||
TokenPoolerOutput: TypeAlias = list[torch.Tensor | None]
|
||||
|
||||
|
||||
class TokenPooler(Pooler):
|
||||
"""A layer that pools specific information from hidden states.
|
||||
|
||||
This layer does the following:
|
||||
1. Extracts specific tokens or aggregates data based on pooling method.
|
||||
2. Postprocesses the output based on pooling head.
|
||||
3. Returns structured results as `PoolerOutput`.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pooling: TokenPoolingMethod | TokenPoolingFn,
|
||||
head: TokenPoolerHead | TokenPoolingHeadFn | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.pooling = pooling
|
||||
self.head = head
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
head_name = self.head.__class__.__name__ if self.head is not None else None
|
||||
return f"pooling={self.pooling.__class__.__name__}, head={head_name}"
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
tasks = set(POOLING_TASKS)
|
||||
|
||||
if isinstance(self.pooling, TokenPoolingMethod):
|
||||
tasks &= self.pooling.get_supported_tasks()
|
||||
if isinstance(self.head, TokenPoolerHead):
|
||||
tasks &= self.head.get_supported_tasks()
|
||||
|
||||
return tasks
|
||||
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
updates = PoolingParamsUpdate()
|
||||
|
||||
if isinstance(self.pooling, TokenPoolingMethod):
|
||||
updates |= self.pooling.get_pooling_updates(task)
|
||||
|
||||
return updates
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> TokenPoolerOutput:
|
||||
pooled_data = self.pooling(hidden_states, pooling_metadata)
|
||||
if self.head is not None:
|
||||
pooled_data = self.head(pooled_data, pooling_metadata)
|
||||
return pooled_data
|
||||
|
||||
|
||||
def pooler_for_token_embed(
|
||||
pooler_config: PoolerConfig, projector: ProjectorFn | None = None
|
||||
) -> TokenPooler:
|
||||
pooling = get_tok_pooling_method(pooler_config.get_tok_pooling_type())
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
model_config = vllm_config.model_config
|
||||
head = TokenEmbeddingPoolerHead(
|
||||
head_dtype=model_config.head_dtype,
|
||||
projector=projector
|
||||
if projector is not None
|
||||
else _load_st_projector(model_config),
|
||||
activation=PoolerNormalize(),
|
||||
)
|
||||
|
||||
return TokenPooler(pooling=pooling, head=head)
|
||||
|
||||
|
||||
def pooler_for_token_classify(
|
||||
pooler_config: PoolerConfig,
|
||||
*,
|
||||
pooling: TokenPoolingMethod | TokenPoolingFn | None = None,
|
||||
classifier: ClassifierFn | None = None,
|
||||
act_fn: PoolerActivation | None = None,
|
||||
) -> TokenPooler:
|
||||
if pooling is None:
|
||||
pooling = get_tok_pooling_method(pooler_config.get_tok_pooling_type())
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
model_config = vllm_config.model_config
|
||||
if model_config.pooler_config is None:
|
||||
raise ValueError(
|
||||
"model_config.pooler_config must be set for token classification pooling"
|
||||
)
|
||||
head = TokenClassifierPoolerHead(
|
||||
head_dtype=model_config.head_dtype,
|
||||
classifier=classifier,
|
||||
logit_mean=model_config.pooler_config.logit_mean,
|
||||
logit_sigma=model_config.pooler_config.logit_sigma,
|
||||
activation=resolve_classifier_act_fn(
|
||||
model_config, static_num_labels=False, act_fn=act_fn
|
||||
),
|
||||
)
|
||||
|
||||
return TokenPooler(pooling=pooling, head=head)
|
||||
-294
@@ -1,294 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
from aiter.ops.shuffle import shuffle_weight
|
||||
from compressed_tensors.quantization import (
|
||||
QuantizationArgs,
|
||||
)
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
int4_w4a16_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
|
||||
CompressedTensorsMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _pack_shuffled_int8_to_packed_int4_no_perm(x_shuf_i8: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack a preshuffled int8 tensor (values in [-8, 7]) into packed int4 bytes.
|
||||
Each contiguous 8-value block [v0..v7] -> 4 bytes:
|
||||
b0=(v4<<4)|v0, b1=(v5<<4)|v1, b2=(v6<<4)|v2, b3=(v7<<4)|v3.
|
||||
This matches the 7-op in-kernel unpack sequence and avoids any v_perm.
|
||||
"""
|
||||
flat = x_shuf_i8.contiguous().view(-1).to(torch.int16)
|
||||
assert flat.numel() % 8 == 0
|
||||
u = (flat & 0xF).to(torch.uint8).view(-1, 8)
|
||||
out = torch.empty((u.shape[0], 4), device=u.device, dtype=torch.uint8)
|
||||
out[:, 0] = u[:, 0] | (u[:, 4] << 4)
|
||||
out[:, 1] = u[:, 1] | (u[:, 5] << 4)
|
||||
out[:, 2] = u[:, 2] | (u[:, 6] << 4)
|
||||
out[:, 3] = u[:, 3] | (u[:, 7] << 4)
|
||||
return out.view(-1).to(torch.int8)
|
||||
|
||||
|
||||
def _unpack_gptq_int32_to_signed_int4(w_int32):
|
||||
"""Unpack GPTQ int32 [E, K//8, N] to signed int4 values [E, N, K] (as int8).
|
||||
Shared by both the packed-int4 and bf16-dequant paths.
|
||||
"""
|
||||
E = w_int32.shape[0]
|
||||
# [E, K//8, N] -> transpose -> [E, N, K//8]
|
||||
w = w_int32.transpose(1, 2).contiguous()
|
||||
N = w.shape[1]
|
||||
K_div8 = w.shape[2]
|
||||
K = K_div8 * 8
|
||||
|
||||
# Unpack int32 -> 8 x uint4 values along K
|
||||
w_expanded = w.unsqueeze(-1).expand(E, N, K_div8, 8) # [E, N, K//8, 8]
|
||||
shifts = torch.arange(8, device=w.device) * 4 # [0, 4, 8, ..., 28]
|
||||
nibbles = ((w_expanded >> shifts) & 0xF).to(torch.int8) # [E, N, K//8, 8]
|
||||
nibbles = nibbles.reshape(E, N, K) # [E, N, K] unsigned int4 as int8
|
||||
|
||||
# Convert unsigned [0,15] to signed [-8,7]
|
||||
signed = nibbles.to(torch.int16) - 8
|
||||
signed = signed.to(torch.int8) # [E, N, K] signed int4 as int8
|
||||
return signed
|
||||
|
||||
|
||||
def _gptq_int32_to_flydsl_packed(w_int32):
|
||||
"""Convert GPTQ int32 [E, K//8, N] to FlyDSL shuffled packed int4 [E, N, K//2].
|
||||
Steps:
|
||||
1. Unpack int32 to individual signed int4 values (as int8)
|
||||
2. Apply FlyDSL preshuffle (on individual int8 values)
|
||||
3. Pack with FlyDSL's interleaved int4 packing
|
||||
"""
|
||||
signed = _unpack_gptq_int32_to_signed_int4(w_int32)
|
||||
E, N, K = signed.shape
|
||||
|
||||
# FlyDSL preshuffle (operates on individual values)
|
||||
shuffled = shuffle_weight(signed, layout=(16, 16))
|
||||
|
||||
# FlyDSL interleaved int4 packing
|
||||
packed = _pack_shuffled_int8_to_packed_int4_no_perm(shuffled).contiguous()
|
||||
return packed.view(E, N, K // 2)
|
||||
|
||||
|
||||
class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant: QuantizationArgs,
|
||||
input_quant: QuantizationArgs | None,
|
||||
moe: FusedMoEConfig,
|
||||
layer_name: str | None = None,
|
||||
):
|
||||
super().__init__(moe)
|
||||
self.weight_quant = weight_quant
|
||||
self.input_quant = input_quant
|
||||
# Extract properties from weight_quant
|
||||
assert weight_quant.num_bits == 4
|
||||
self.num_bits = weight_quant.num_bits
|
||||
self.packed_factor = 32 // weight_quant.num_bits
|
||||
self.strategy = weight_quant.strategy
|
||||
# channelwise is not supported by this kernel
|
||||
assert weight_quant.strategy == "group"
|
||||
assert weight_quant.group_size == 32
|
||||
self.group_size = weight_quant.group_size
|
||||
assert weight_quant.symmetric, (
|
||||
"Only symmetric quantization is supported for MoE. "
|
||||
"Try --moe-backend emulation."
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
self.num_experts = num_experts
|
||||
self.inter_dim = intermediate_size_per_partition
|
||||
# Will transpose the loaded weight along the
|
||||
# intermediate and hidden dim sizes. Will
|
||||
# shard for TP along the transposed dims
|
||||
extra_weight_attrs.update(
|
||||
{"is_transposed": True, "quant_method": self.strategy}
|
||||
)
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size // self.packed_factor,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_packed", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
intermediate_size_per_partition // self.packed_factor,
|
||||
hidden_size,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_packed", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
w2_scales_size = intermediate_size_per_partition
|
||||
|
||||
if self.strategy == "channel":
|
||||
num_groups_w2 = num_groups_w13 = 1
|
||||
self.group_size = -1
|
||||
else:
|
||||
num_groups_w2 = w2_scales_size // self.group_size
|
||||
num_groups_w13 = hidden_size // self.group_size
|
||||
|
||||
w13_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
num_groups_w13,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_scale)
|
||||
set_weight_attrs(w13_scale, extra_weight_attrs)
|
||||
|
||||
w2_scale = torch.nn.Parameter(
|
||||
torch.ones(num_experts, num_groups_w2, hidden_size, dtype=params_dtype),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_scale)
|
||||
set_weight_attrs(w2_scale, extra_weight_attrs)
|
||||
|
||||
w2_weight_shape = torch.nn.Parameter(
|
||||
torch.empty(num_experts, 2), requires_grad=False
|
||||
)
|
||||
layer.register_parameter("w2_weight_shape", w2_weight_shape)
|
||||
set_weight_attrs(w2_weight_shape, extra_weight_attrs)
|
||||
w13_weight_shape = torch.nn.Parameter(
|
||||
torch.empty(num_experts, 2), requires_grad=False
|
||||
)
|
||||
|
||||
layer.register_parameter("w13_weight_shape", w13_weight_shape)
|
||||
set_weight_attrs(w13_weight_shape, extra_weight_attrs)
|
||||
|
||||
layer.a13_scale = None
|
||||
layer.a2_scale = None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Reconfigure packed weights and scales to match flydsl_w4a16 format
|
||||
|
||||
# Convert w13 weights
|
||||
w13 = layer.w13_weight_packed.data
|
||||
w13 = _gptq_int32_to_flydsl_packed(w13)
|
||||
w13 = w13.view(-1).contiguous()
|
||||
layer.w13_weight_packed = torch.nn.Parameter(w13, requires_grad=False)
|
||||
|
||||
# Convert w2 weights
|
||||
w2 = layer.w2_weight_packed.data
|
||||
w2 = _gptq_int32_to_flydsl_packed(w2)
|
||||
w2 = w2.view(-1).contiguous()
|
||||
layer.w2_weight_packed = torch.nn.Parameter(w2, requires_grad=False)
|
||||
|
||||
# Convert scales for FlyDSL:
|
||||
# per-row: [E, 1, N] -> squeeze -> [E, N]
|
||||
# groupwise: [E, K//gs, N] -> keep as-is (Opt 0: cache-friendly layout)
|
||||
w13_scale = layer.w13_weight_scale.data
|
||||
if self.group_size > 0 and w13_scale.dim() == 3 and w13_scale.shape[1] > 1:
|
||||
E, G, N = w13_scale.shape
|
||||
w13_scale = (
|
||||
w13_scale.view(E, G // 2, 2, N)
|
||||
.permute(0, 1, 3, 2)
|
||||
.contiguous()
|
||||
.view(-1)
|
||||
.contiguous()
|
||||
)
|
||||
elif w13_scale.dim() == 3 and w13_scale.shape[1] == 1:
|
||||
# Per-row: squeeze [E, 1, N] -> [E, N]
|
||||
w13_scale = w13_scale.squeeze(1)
|
||||
layer.w13_weight_scale = torch.nn.Parameter(
|
||||
w13_scale.contiguous(), requires_grad=False
|
||||
)
|
||||
|
||||
w2_scale = layer.w2_weight_scale.data
|
||||
if self.group_size > 0 and w2_scale.dim() == 3 and w2_scale.shape[1] > 1:
|
||||
E, G, N = w2_scale.shape
|
||||
w2_scale = (
|
||||
w2_scale.view(E, G // 2, 2, N)
|
||||
.permute(0, 1, 3, 2)
|
||||
.contiguous()
|
||||
.view(-1)
|
||||
.contiguous()
|
||||
)
|
||||
elif w2_scale.dim() == 3 and w2_scale.shape[1] == 1:
|
||||
# Per-row: squeeze [E, 1, N] -> [E, N]
|
||||
w2_scale = w2_scale.squeeze(1)
|
||||
layer.w2_weight_scale = torch.nn.Parameter(
|
||||
w2_scale.contiguous(), requires_grad=False
|
||||
)
|
||||
|
||||
layer.w13_weight_packed.is_shuffled = True
|
||||
layer.w2_weight_packed.is_shuffled = True
|
||||
layer.is_aiter_converted = True
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
assert self.num_bits == 4
|
||||
return int4_w4a16_moe_quant_config(
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w1_zp=None,
|
||||
w2_zp=None,
|
||||
block_shape=[0, self.group_size],
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.fused_moe.fused_flydsl_moe import (
|
||||
fused_flydsl_moe,
|
||||
)
|
||||
|
||||
assert self.moe_quant_config is not None
|
||||
|
||||
return fused_flydsl_moe(
|
||||
x,
|
||||
layer.w13_weight_packed,
|
||||
layer.w2_weight_packed,
|
||||
self.num_experts,
|
||||
self.inter_dim,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
w1_scale=self.moe_quant_config.w1_scale,
|
||||
w2_scale=self.moe_quant_config.w2_scale,
|
||||
topk=topk_weights.shape[-1],
|
||||
group_size=self.group_size,
|
||||
doweight_stage1=layer.apply_router_weight_on_input,
|
||||
scale_is_bf16=True,
|
||||
)
|
||||
-272
@@ -1,272 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
mxfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsMxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
MarlinExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
|
||||
XPUExpertsMxFp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
|
||||
is_flashinfer_moe_ep_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
B12X_BACKENDS,
|
||||
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
convert_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
make_mxfp4_moe_quant_config,
|
||||
select_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
|
||||
CompressedTensorsMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
prepare_moe_fp4_layer_for_marlin,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
def __init__(self, moe):
|
||||
super().__init__(moe)
|
||||
self.group_size = 32
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None = None
|
||||
|
||||
# Backend selection must match the weight preparation below: CUTLASS
|
||||
# swizzles scales, b12x and XPU consume checkpoint packing, Marlin
|
||||
# repacks weights and scales, and the megakernels take the checkpoint
|
||||
# tensors as they are.
|
||||
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
|
||||
if moe.moe_backend == "b12x" or is_flashinfer_moe_ep_backend(moe.moe_backend):
|
||||
self.mxfp4_backend, experts_cls = select_mxfp4_moe_backend(moe)
|
||||
assert experts_cls is not None
|
||||
self.experts_cls = experts_cls
|
||||
self.use_cutlass_mxfp4 = False
|
||||
elif self.use_cutlass_mxfp4:
|
||||
logger.info_once("Using CutlassExpertsMxfp4 for MXFP4 MoE")
|
||||
self.experts_cls = CutlassExpertsMxfp4
|
||||
elif current_platform.is_xpu():
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.XPU
|
||||
self.experts_cls = XPUExpertsMxFp4
|
||||
logger.info_once("Using XPUExpertsMxFp4 for MXFP4 MoE on XPU platform")
|
||||
else:
|
||||
logger.info_once("Using MarlinExperts for MXFP4 MoE")
|
||||
self.experts_cls = MarlinExperts
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
return self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
layer.num_experts = num_experts
|
||||
layer.params_dtype = params_dtype
|
||||
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
# 2 fp4 items are packed in the input dimension
|
||||
hidden_size // 2,
|
||||
requires_grad=False,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_packed", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
# 2 fp4 items are packed in the input dimension
|
||||
intermediate_size_per_partition // 2,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_packed", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
# 2 fp4 items are packed in the input dimension
|
||||
hidden_size // self.group_size,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
extra_weight_attrs.update(
|
||||
{"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
|
||||
)
|
||||
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
|
||||
|
||||
w2_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
# 2 fp4 items are packed in the input dimension
|
||||
intermediate_size_per_partition // self.group_size,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
if self.use_cutlass_mxfp4:
|
||||
# W4A4: both weights and activations quantized to MXFP4
|
||||
return mxfp4_moe_quant_config(
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
)
|
||||
else:
|
||||
# b12x selects W4A8 or W4A16; XPU uses W4A4; Marlin uses W4A16.
|
||||
return make_mxfp4_moe_quant_config(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
layer.w13_weight = torch.nn.Parameter(
|
||||
layer.w13_weight_packed.data, requires_grad=False
|
||||
)
|
||||
delattr(layer, "w13_weight_packed")
|
||||
|
||||
layer.w2_weight = torch.nn.Parameter(
|
||||
layer.w2_weight_packed.data, requires_grad=False
|
||||
)
|
||||
delattr(layer, "w2_weight_packed")
|
||||
|
||||
if self.use_cutlass_mxfp4:
|
||||
# Swizzle weight scales from flat checkpoint layout [E, N, K//32]
|
||||
# to CUTLASS tiled layout [E, numMTiles*numKTiles*512].
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
swizzle_mxfp4_scales,
|
||||
)
|
||||
|
||||
E = layer.w13_weight_scale.shape[0]
|
||||
w13_N = layer.w13_weight_scale.shape[1]
|
||||
w13_scale_K = layer.w13_weight_scale.shape[2]
|
||||
w13_K = w13_scale_K * 32
|
||||
|
||||
w2_M = layer.w2_weight_scale.shape[1]
|
||||
w2_scale_N = layer.w2_weight_scale.shape[2]
|
||||
w2_N = w2_scale_N * 32
|
||||
|
||||
swizzled_w13 = []
|
||||
swizzled_w2 = []
|
||||
for e_idx in range(E):
|
||||
s13 = layer.w13_weight_scale[e_idx]
|
||||
sw13 = swizzle_mxfp4_scales(s13, w13_N, w13_K)
|
||||
swizzled_w13.append(sw13.reshape(w13_N, w13_scale_K))
|
||||
s2 = layer.w2_weight_scale[e_idx]
|
||||
sw2 = swizzle_mxfp4_scales(s2, w2_M, w2_N)
|
||||
swizzled_w2.append(sw2.reshape(w2_M, w2_scale_N))
|
||||
layer.w13_weight_scale = torch.nn.Parameter(
|
||||
torch.stack(swizzled_w13), requires_grad=False
|
||||
)
|
||||
layer.w2_weight_scale = torch.nn.Parameter(
|
||||
torch.stack(swizzled_w2), requires_grad=False
|
||||
)
|
||||
elif self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
w13, w2, w13_scale, w2_scale, _, _ = (
|
||||
convert_weight_to_mxfp4_moe_kernel_format(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
layer=layer,
|
||||
w13_weight=layer.w13_weight,
|
||||
w2_weight=layer.w2_weight,
|
||||
w13_weight_scale=layer.w13_weight_scale,
|
||||
w2_weight_scale=layer.w2_weight_scale,
|
||||
)
|
||||
)
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
|
||||
pass
|
||||
else:
|
||||
logger.warning_once(
|
||||
"Your GPU does not have native support for FP4 computation "
|
||||
"but FP4 quantization is being used. Weight-only FP4 "
|
||||
"compression will be used leveraging the Marlin kernel. "
|
||||
"This may degrade performance for compute-heavy workloads."
|
||||
)
|
||||
prepare_moe_fp4_layer_for_marlin(layer)
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None:
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
|
||||
|
||||
_PACKED_WEIGHTS = ("w13_weight", "w2_weight")
|
||||
|
||||
|
||||
class CompressedTensorsW4A8Mxfp4MoEMethod(Mxfp4MoEMethod):
|
||||
"""MXFP4 weights with dynamic FP8 activations in compressed-tensors format.
|
||||
|
||||
Only the checkpoint naming and the declared activation scheme differ from
|
||||
the native MXFP4 path, so backend selection, weight conversion and
|
||||
execution are inherited.
|
||||
"""
|
||||
|
||||
# The packed -> unpacked rename means a processed state dict no longer
|
||||
# matches the parameters registered by create_weights.
|
||||
supports_pre_processed_weights = False
|
||||
|
||||
def create_weights(self, layer: RoutedExperts, *args, **kwargs):
|
||||
super().create_weights(layer, *args, **kwargs)
|
||||
for name in _PACKED_WEIGHTS:
|
||||
param = getattr(layer, name)
|
||||
delattr(layer, name)
|
||||
layer.register_parameter(f"{name}_packed", param)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
for name in _PACKED_WEIGHTS:
|
||||
param = getattr(layer, f"{name}_packed")
|
||||
delattr(layer, f"{name}_packed")
|
||||
layer.register_parameter(name, param)
|
||||
super().process_weights_after_loading(layer)
|
||||
-208
@@ -1,208 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe import ( # noqa: E501
|
||||
CompressedTensorsMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
|
||||
|
||||
class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
"""Compressed-tensors MoE method for pre-quantized MXFP8 (W8A8) checkpoints.
|
||||
|
||||
Loads FP8 (E4M3) weights with E8M0 uint8 per-group scales (group_size=32)
|
||||
from checkpoint. Activations are dynamically quantized to MXFP8 at runtime.
|
||||
Supports FlashInfer TRT-LLM and Marlin backends (auto-selected).
|
||||
"""
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.weight_block_size = [1, MXFP8_BLOCK_SIZE]
|
||||
self.fp8_backend, self.experts_cls = select_mxfp8_moe_backend(config=self.moe)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
layer.num_experts = num_experts
|
||||
layer.params_dtype = params_dtype
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // MXFP8_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
extra_weight_attrs.update(
|
||||
{"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
|
||||
)
|
||||
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
|
||||
|
||||
w2_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // MXFP8_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
|
||||
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
|
||||
w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format(
|
||||
fp8_backend=self.fp8_backend,
|
||||
layer=layer,
|
||||
w13=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w13_input_scale=layer.w13_input_scale,
|
||||
w2_input_scale=layer.w2_input_scale,
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None:
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_fp8_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
return make_fp8_moe_quant_config(
|
||||
fp8_backend=self.fp8_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
a1_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
router_logits,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
num_expert_group=layer.num_expert_group,
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
@@ -5,8 +5,8 @@ from .compressed_tensors_scheme import CompressedTensorsScheme
|
||||
from .compressed_tensors_w4a4_mxfp4 import CompressedTensorsW4A4Mxfp4
|
||||
from .compressed_tensors_w4a4_nvfp4 import CompressedTensorsW4A4Fp4
|
||||
from .compressed_tensors_w4a8_fp8 import CompressedTensorsW4A8Fp8
|
||||
from .compressed_tensors_w4a8_int import CompressedTensorsW4A8Int
|
||||
from .compressed_tensors_w4a8_mxfp4 import CompressedTensorsW4A8Mxfp4
|
||||
from .compressed_tensors_w4a8_int import CompressedTensorsW4A8Int
|
||||
from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
|
||||
from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
|
||||
from .compressed_tensors_w8a8_mxfp8 import CompressedTensorsW8A8Mxfp8
|
||||
@@ -23,12 +23,12 @@ __all__ = [
|
||||
"CompressedTensorsW8A16Fp8",
|
||||
"CompressedTensorsW8A8Int8",
|
||||
"CompressedTensorsW8A8Fp8",
|
||||
"CompressedTensorsW4A4Fp4",
|
||||
"CompressedTensorsW4A4Mxfp4",
|
||||
"CompressedTensorsW4A8Mxfp4",
|
||||
"CompressedTensorsW4A4Fp4",
|
||||
"CompressedTensorsW8A8Mxfp8",
|
||||
"CompressedTensorsW4A8Int",
|
||||
"CompressedTensorsW4A8Fp8",
|
||||
"CompressedTensorsW8A8Mxfp8",
|
||||
"CompressedTensorsWNA4Int",
|
||||
"CompressedTensorsWNA8Int",
|
||||
]
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""TurboQuant: KV-cache quantization for vLLM.
|
||||
|
||||
Hadamard rotation + per-coordinate Lloyd-Max scalar quantization for
|
||||
keys, uniform quantization for values.
|
||||
|
||||
The core algorithmic pattern implemented for key quantization (Hadamard
|
||||
rotation followed by deterministic scalar quantization and
|
||||
re-normalization) was originally established in DRIVE (Vargaftik et al.,
|
||||
NeurIPS 2021) and EDEN (Vargaftik et al., ICML 2022). This formulation is
|
||||
also mathematically equivalent to the scalar case of the HIGGS
|
||||
quantization method (Malinovskii et al., "Pushing the Limits of Large
|
||||
Language Model Quantization via the Linearity Theorem", NAACL 2025;
|
||||
preprint arXiv:2411.17525), which subsequently generalized these concepts.
|
||||
|
||||
A first application of this approach to KV-cache compression is in "Cache
|
||||
Me If You Must: Adaptive Key-Value Quantization for Large Language Models"
|
||||
(Shutova et al., ICML 2025; preprint arXiv:2501.19392). All of these
|
||||
foundational and application references pre-date the TurboQuant paper
|
||||
(Zandieh et al., ICLR 2026).
|
||||
"""
|
||||
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import TurboQuantConfig
|
||||
|
||||
__all__ = ["TurboQuantConfig"]
|
||||
@@ -1,87 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Lloyd-Max optimal scalar quantizer for TurboQuant.
|
||||
|
||||
After rotating a d-dimensional unit vector by a random orthogonal matrix,
|
||||
each coordinate approximately follows N(0, 1/d) for d >= 64.
|
||||
We solve the Lloyd-Max conditions to find optimal centroids.
|
||||
|
||||
Based on: turboquant-pytorch/lloyd_max.py (Zandieh et al.)
|
||||
"""
|
||||
|
||||
import math
|
||||
from functools import lru_cache
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def _gaussian_pdf(x: float, sigma2: float) -> float:
|
||||
return (1.0 / math.sqrt(2 * math.pi * sigma2)) * math.exp(-x * x / (2 * sigma2))
|
||||
|
||||
|
||||
def _trapz(f, a: float, b: float, n: int = 200) -> float:
|
||||
"""Trapezoidal numerical integration (replaces scipy.integrate.quad)."""
|
||||
h = (b - a) / n
|
||||
result = 0.5 * (f(a) + f(b))
|
||||
for i in range(1, n):
|
||||
result += f(a + i * h)
|
||||
return result * h
|
||||
|
||||
|
||||
def solve_lloyd_max(
|
||||
d: int,
|
||||
bits: int,
|
||||
max_iter: int = 200,
|
||||
tol: float = 1e-10,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Solve Lloyd-Max optimal quantizer for N(0, 1/d) distribution.
|
||||
|
||||
Args:
|
||||
d: Vector dimension (determines variance = 1/d).
|
||||
bits: Number of quantization bits.
|
||||
max_iter: Maximum Lloyd-Max iterations.
|
||||
tol: Convergence tolerance.
|
||||
|
||||
Returns:
|
||||
centroids: Sorted tensor of 2^bits optimal centroids.
|
||||
boundaries: Sorted tensor of 2^bits - 1 decision boundaries.
|
||||
|
||||
"""
|
||||
n_levels = 2**bits
|
||||
sigma2 = 1.0 / d
|
||||
sigma = math.sqrt(sigma2)
|
||||
|
||||
def pdf(x):
|
||||
return _gaussian_pdf(x, sigma2)
|
||||
|
||||
lo, hi = -3.5 * sigma, 3.5 * sigma
|
||||
centroids = [lo + (hi - lo) * (i + 0.5) / n_levels for i in range(n_levels)]
|
||||
|
||||
for _ in range(max_iter):
|
||||
boundaries = [
|
||||
(centroids[i] + centroids[i + 1]) / 2.0 for i in range(n_levels - 1)
|
||||
]
|
||||
edges = [lo * 3] + boundaries + [hi * 3]
|
||||
new_centroids = []
|
||||
for i in range(n_levels):
|
||||
a, b = edges[i], edges[i + 1]
|
||||
num = _trapz(lambda x: x * pdf(x), a, b)
|
||||
den = _trapz(pdf, a, b)
|
||||
new_centroids.append(num / den if den > 1e-15 else centroids[i])
|
||||
|
||||
if max(abs(new_centroids[i] - centroids[i]) for i in range(n_levels)) < tol:
|
||||
break
|
||||
centroids = new_centroids
|
||||
|
||||
boundaries = [(centroids[i] + centroids[i + 1]) / 2.0 for i in range(n_levels - 1)]
|
||||
return (
|
||||
torch.tensor(centroids, dtype=torch.float32),
|
||||
torch.tensor(boundaries, dtype=torch.float32),
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=32)
|
||||
def get_centroids(d: int, bits: int) -> torch.Tensor:
|
||||
"""Get precomputed Lloyd-Max centroids (cached)."""
|
||||
centroids, _ = solve_lloyd_max(d, bits)
|
||||
return centroids
|
||||
@@ -1,261 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""TurboQuant configuration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Named TQ presets: each maps to frozen config parameters.
|
||||
# key_quant_bits: 8 = FP8 keys, 3-4 = MSE (Lloyd-Max) quantized keys.
|
||||
# value_quant_bits: 3-4 = uniform quantized values.
|
||||
TQ_PRESETS: dict[str, dict] = {
|
||||
"turboquant_k8v4": {
|
||||
"key_quant_bits": 8,
|
||||
"value_quant_bits": 4,
|
||||
"norm_correction": False,
|
||||
},
|
||||
"turboquant_4bit_nc": {
|
||||
"key_quant_bits": 4,
|
||||
"value_quant_bits": 4,
|
||||
"norm_correction": True,
|
||||
},
|
||||
"turboquant_k3v4_nc": {
|
||||
"key_quant_bits": 3,
|
||||
"value_quant_bits": 4,
|
||||
"norm_correction": True,
|
||||
},
|
||||
"turboquant_3bit_nc": {
|
||||
"key_quant_bits": 3,
|
||||
"value_quant_bits": 3,
|
||||
"norm_correction": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class TurboQuantConfig:
|
||||
"""Configuration for TurboQuant KV-cache quantization.
|
||||
|
||||
Applies Hadamard rotation followed by per-coordinate Lloyd-Max scalar
|
||||
quantization for keys, and uniform quantization for values.
|
||||
|
||||
Historical note: the core algorithmic pattern implemented for key
|
||||
quantization (Hadamard rotation followed by deterministic scalar
|
||||
quantization and re-normalization) was originally established in DRIVE
|
||||
(Vargaftik et al., NeurIPS 2021) and EDEN (Vargaftik et al., ICML
|
||||
2022). This formulation is also mathematically equivalent to the
|
||||
scalar case of the HIGGS quantization method (Malinovskii et al.,
|
||||
"Pushing the Limits of Large Language Model Quantization via the
|
||||
Linearity Theorem", NAACL 2025; preprint arXiv:2411.17525), which
|
||||
subsequently generalized these concepts.
|
||||
|
||||
A first application of this approach to KV-cache compression is in
|
||||
"Cache Me If You Must: Adaptive Key-Value Quantization for Large
|
||||
Language Models" (Shutova et al., ICML 2025; preprint
|
||||
arXiv:2501.19392). All of these foundational and application
|
||||
references pre-date the TurboQuant paper (Zandieh et al., ICLR 2026).
|
||||
|
||||
QJL is intentionally omitted: community consensus (5+ independent
|
||||
groups) found it hurts attention quality by amplifying variance
|
||||
through softmax.
|
||||
|
||||
Named presets (use via --kv-cache-dtype):
|
||||
turboquant_k8v4: FP8 keys + 4-bit values, 2.6x, +1.17% PPL
|
||||
turboquant_4bit_nc: 4-bit MSE keys + 4-bit values + NC, 3.8x, +2.71%
|
||||
turboquant_k3v4_nc: 3-bit MSE keys + 4-bit values + NC, ~3.5x, +10.63%
|
||||
turboquant_3bit_nc: 3-bit MSE keys + 3-bit values + NC, 4.9x, +20.59%
|
||||
|
||||
Args:
|
||||
head_dim: Attention head dimension (e.g. 64, 96, 128).
|
||||
key_quant_bits: Bits for key quantization. 8 = FP8 keys (no
|
||||
rotation/MSE). 3-4 = Lloyd-Max MSE quantized keys.
|
||||
value_quant_bits: Bits per value dimension for uniform quantization.
|
||||
3 = 8 levels, 4 = 16 levels (default).
|
||||
norm_correction: Re-normalize centroid vectors to unit norm before
|
||||
inverse rotation during dequant. Fixes quantization-induced norm
|
||||
distortion, improving PPL by ~0.8% at 4-bit.
|
||||
|
||||
"""
|
||||
|
||||
head_dim: int = 128
|
||||
key_quant_bits: int = 3 # 3-4 = MSE keys, 8 = FP8 keys
|
||||
value_quant_bits: int = 4 # 3-4 = uniform quantized values
|
||||
seed: int = 42 # kept for backward compatibility; no longer used internally
|
||||
norm_correction: bool = False
|
||||
|
||||
@property
|
||||
def key_fp8(self) -> bool:
|
||||
"""Whether keys are stored as FP8 — no rotation/quantization needed."""
|
||||
return self.key_quant_bits == 8
|
||||
|
||||
@property
|
||||
def mse_bits(self) -> int:
|
||||
"""MSE quantizer bit-width (determines centroid count: 2^mse_bits).
|
||||
|
||||
For MSE key modes, equals key_quant_bits.
|
||||
For FP8 key mode, falls back to value_quant_bits (centroids are still
|
||||
needed for continuation-prefill dequant and decode kernel params).
|
||||
"""
|
||||
if self.key_fp8:
|
||||
return self.value_quant_bits
|
||||
return self.key_quant_bits
|
||||
|
||||
@property
|
||||
def key_mse_bits(self) -> int:
|
||||
"""MSE bits actually used for key quantization (0 if FP8 keys)."""
|
||||
if self.key_fp8:
|
||||
return 0
|
||||
return self.key_quant_bits
|
||||
|
||||
@property
|
||||
def centroid_bits(self) -> int:
|
||||
"""Bits for centroid generation — always non-zero."""
|
||||
return self.mse_bits
|
||||
|
||||
@property
|
||||
def n_centroids(self) -> int:
|
||||
return 2**self.mse_bits
|
||||
|
||||
@property
|
||||
def key_packed_size(self) -> int:
|
||||
"""Packed bytes for a single KEY vector.
|
||||
|
||||
FP8 mode (key_quant_bits=8):
|
||||
head_dim bytes (1 byte per element, no overhead).
|
||||
|
||||
TQ mode:
|
||||
- MSE indices: ceil(head_dim * key_mse_bits / 8) bytes
|
||||
- vec_norm: 2 bytes (float16)
|
||||
"""
|
||||
if self.key_fp8:
|
||||
return self.head_dim # 1 byte per element
|
||||
mse_bytes = math.ceil(self.head_dim * self.key_mse_bits / 8)
|
||||
norm_bytes = 2 # vec_norm fp16
|
||||
return mse_bytes + norm_bytes
|
||||
|
||||
@property
|
||||
def effective_value_quant_bits(self) -> int:
|
||||
"""Actual bits used for value storage."""
|
||||
return self.value_quant_bits
|
||||
|
||||
@property
|
||||
def value_packed_size(self) -> int:
|
||||
"""Packed bytes for a single VALUE vector.
|
||||
|
||||
Uniform quantization: ceil(head_dim * bits / 8) + 4 bytes (scale + zero fp16).
|
||||
"""
|
||||
data_bytes = math.ceil(self.head_dim * self.value_quant_bits / 8)
|
||||
return data_bytes + 4 # +2 scale(fp16) +2 zero(fp16)
|
||||
|
||||
@property
|
||||
def slot_size(self) -> int:
|
||||
"""Total packed bytes per head per position (key + value combined).
|
||||
|
||||
Layout: [key_packed | value_packed]
|
||||
"""
|
||||
return self.key_packed_size + self.value_packed_size
|
||||
|
||||
@property
|
||||
def slot_size_aligned(self) -> int:
|
||||
"""Slot size rounded up to next even number.
|
||||
|
||||
Even-number is required so effective_head_size = slot_size_aligned // 2
|
||||
is integral.
|
||||
"""
|
||||
s = self.slot_size
|
||||
return s + (s % 2) # round up to even
|
||||
|
||||
@staticmethod
|
||||
def get_boundary_skip_layers(
|
||||
model_config: ModelConfig,
|
||||
n: int = 2,
|
||||
) -> list[str]:
|
||||
"""Layer indices to skip TQ compression (boundary protection).
|
||||
|
||||
For hybrid models (attention + Mamba/linear-attention), boundary
|
||||
protection is disabled — hybrids typically have only 8-12
|
||||
full-attention layers and a hard n=2 on each side would cover
|
||||
~40 % of them. The dense GSM8K baselines that motivate n=2
|
||||
don't apply to hybrids.
|
||||
|
||||
For dense models, skips first N and last N attention layers.
|
||||
Empirically required for aggressive presets (k3v4_nc, 3bit_nc)
|
||||
— without it GSM8K drops ~30 points on Qwen3-4B.
|
||||
"""
|
||||
if model_config.is_hybrid:
|
||||
attn_indices = _get_full_attention_layer_indices(model_config)
|
||||
if not attn_indices:
|
||||
raise NotImplementedError(
|
||||
"TurboQuant KV cache requires identifiable "
|
||||
"full-attention layers, but none were found in "
|
||||
"the hybrid model config."
|
||||
)
|
||||
logger.info("TQ hybrid: full-attention layers %s", attn_indices)
|
||||
return []
|
||||
|
||||
num_layers = model_config.hf_text_config.num_hidden_layers
|
||||
if n <= 0 or num_layers <= 0:
|
||||
return []
|
||||
n = min(n, num_layers // 2) # don't skip more than half
|
||||
first = list(range(n))
|
||||
last = list(range(num_layers - n, num_layers))
|
||||
# Deduplicate (if num_layers <= 2*n)
|
||||
indices = sorted(set(first + last))
|
||||
return [str(i) for i in indices]
|
||||
|
||||
@staticmethod
|
||||
def from_cache_dtype(cache_dtype: str, head_dim: int) -> TurboQuantConfig:
|
||||
"""Create config from a named preset.
|
||||
|
||||
Valid presets: turboquant_k8v4, turboquant_4bit_nc, etc.
|
||||
"""
|
||||
if cache_dtype not in TQ_PRESETS:
|
||||
valid = ", ".join(TQ_PRESETS.keys())
|
||||
raise ValueError(
|
||||
f"Unknown TurboQuant cache dtype: {cache_dtype!r}. "
|
||||
f"Valid presets: {valid}"
|
||||
)
|
||||
preset = TQ_PRESETS[cache_dtype]
|
||||
return TurboQuantConfig(
|
||||
head_dim=head_dim,
|
||||
key_quant_bits=preset["key_quant_bits"],
|
||||
value_quant_bits=preset["value_quant_bits"],
|
||||
norm_correction=preset["norm_correction"],
|
||||
)
|
||||
|
||||
|
||||
def _get_full_attention_layer_indices(model_config: ModelConfig) -> list[int]:
|
||||
"""Global indices of full-attention layers in a hybrid model.
|
||||
|
||||
Covers the conventions used across vLLM: ``layer_types`` (Qwen3.5/Next),
|
||||
``layers_block_type`` (Jamba/Zamba2), ``attn_type_list`` (Minimax).
|
||||
"""
|
||||
text_cfg = model_config.hf_text_config
|
||||
hf_cfg = model_config.hf_config
|
||||
|
||||
layer_types = getattr(text_cfg, "layer_types", None)
|
||||
if layer_types is not None:
|
||||
return [
|
||||
i for i, t in enumerate(layer_types) if t in ("full_attention", "attention")
|
||||
]
|
||||
|
||||
layers_block_type = getattr(text_cfg, "layers_block_type", None)
|
||||
if layers_block_type is not None:
|
||||
return [
|
||||
i for i, t in enumerate(layers_block_type) if t in ("attention", "hybrid")
|
||||
]
|
||||
|
||||
attn_type_list = getattr(hf_cfg, "attn_type_list", None)
|
||||
if attn_type_list is not None:
|
||||
return [i for i, t in enumerate(attn_type_list) if t == 1]
|
||||
|
||||
return []
|
||||
@@ -1,207 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility functions used for tests and benchmarks."""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.scalar_type import ScalarType, scalar_types
|
||||
|
||||
from .marlin_utils import GPTQ_MARLIN_TILE, marlin_permute_scales, marlin_zero_points
|
||||
from .quant_utils import (
|
||||
get_pack_factor,
|
||||
gptq_quantize_weights,
|
||||
quantize_weights,
|
||||
)
|
||||
|
||||
|
||||
class MarlinWorkspace:
|
||||
def __init__(self, out_features, min_thread_n, max_parallel):
|
||||
assert out_features % min_thread_n == 0, (
|
||||
"out_features = {} is indivisible by min_thread_n = {}".format(
|
||||
out_features, min_thread_n
|
||||
)
|
||||
)
|
||||
|
||||
max_workspace_size = (out_features // min_thread_n) * max_parallel
|
||||
|
||||
self.scratch = torch.zeros(max_workspace_size, dtype=torch.int, device="cuda")
|
||||
|
||||
|
||||
def marlin_permute_weights(
|
||||
q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE, is_a_8bit=False
|
||||
):
|
||||
assert q_w.shape == (size_k, size_n)
|
||||
assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
|
||||
assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
|
||||
|
||||
if is_a_8bit:
|
||||
# Permute weights to 32x32 marlin tiles
|
||||
q_w = q_w.reshape((size_k // (tile * 2), tile * 2, size_n // tile, tile))
|
||||
else:
|
||||
# Permute weights to 16x64 marlin tiles
|
||||
q_w = q_w.reshape((size_k // tile, tile, size_n // tile, tile))
|
||||
q_w = q_w.permute((0, 2, 1, 3))
|
||||
q_w = q_w.reshape((size_k // tile, size_n * tile))
|
||||
|
||||
q_w = q_w.reshape((-1, perm.numel()))[:, perm].reshape(q_w.shape)
|
||||
|
||||
return q_w
|
||||
|
||||
|
||||
def marlin_weights(q_w, size_k, size_n, num_bits, perm, is_a_8bit=False):
|
||||
# Permute
|
||||
q_w = marlin_permute_weights(q_w, size_k, size_n, perm, is_a_8bit=is_a_8bit)
|
||||
|
||||
# Pack
|
||||
pack_factor = get_pack_factor(num_bits)
|
||||
orig_device = q_w.device
|
||||
|
||||
q_w = q_w.cpu().numpy().astype(np.uint32)
|
||||
|
||||
q_packed = np.zeros((q_w.shape[0], q_w.shape[1] // pack_factor), dtype=np.uint32)
|
||||
for i in range(pack_factor):
|
||||
q_packed |= q_w[:, i::pack_factor] << num_bits * i
|
||||
|
||||
q_packed = torch.from_numpy(q_packed.astype(np.int32)).to(orig_device)
|
||||
|
||||
return q_packed
|
||||
|
||||
|
||||
def get_weight_perm(num_bits: int, is_a_8bit: bool = False):
|
||||
perm_list: list[int] = []
|
||||
if is_a_8bit:
|
||||
for i in range(32):
|
||||
perm1 = []
|
||||
col = i // 4
|
||||
for block in [0, 1]:
|
||||
for row in [
|
||||
4 * (i % 4),
|
||||
4 * (i % 4) + 1,
|
||||
4 * (i % 4) + 2,
|
||||
4 * (i % 4) + 3,
|
||||
4 * (i % 4 + 4),
|
||||
4 * (i % 4 + 4) + 1,
|
||||
4 * (i % 4 + 4) + 2,
|
||||
4 * (i % 4 + 4) + 3,
|
||||
]:
|
||||
perm1.append(16 * row + col + 8 * block)
|
||||
for j in range(2):
|
||||
perm_list.extend([p + 512 * j for p in perm1])
|
||||
else:
|
||||
for i in range(32):
|
||||
perm1 = []
|
||||
col = i // 4
|
||||
for block in [0, 1]:
|
||||
for row in [
|
||||
2 * (i % 4),
|
||||
2 * (i % 4) + 1,
|
||||
2 * (i % 4 + 4),
|
||||
2 * (i % 4 + 4) + 1,
|
||||
]:
|
||||
perm1.append(16 * row + col + 8 * block)
|
||||
for j in range(4):
|
||||
perm_list.extend([p + 256 * j for p in perm1])
|
||||
|
||||
perm = np.array(perm_list)
|
||||
|
||||
if num_bits == 4:
|
||||
if is_a_8bit: # noqa: SIM108
|
||||
interleave = np.array([0, 4, 1, 5, 2, 6, 3, 7])
|
||||
else:
|
||||
interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7])
|
||||
elif num_bits == 8:
|
||||
if is_a_8bit: # noqa: SIM108
|
||||
interleave = np.array([0, 1, 2, 3])
|
||||
else:
|
||||
interleave = np.array([0, 2, 1, 3])
|
||||
else:
|
||||
raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
|
||||
|
||||
perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
|
||||
perm = torch.from_numpy(perm)
|
||||
return perm
|
||||
|
||||
|
||||
def marlin_quantize(
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
):
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
size_k, size_n = w.shape
|
||||
num_bits = quant_type.size_bits
|
||||
|
||||
# Normalize group_size
|
||||
if group_size == -1:
|
||||
group_size = size_k
|
||||
assert group_size <= size_k
|
||||
|
||||
w_ref, q_w, s = gptq_quantize_weights(w, quant_type, group_size)
|
||||
|
||||
# Reformat to marlin
|
||||
weight_perm = get_weight_perm(num_bits, is_a_8bit)
|
||||
marlin_q_w = marlin_weights(
|
||||
q_w, size_k, size_n, num_bits, weight_perm, is_a_8bit=is_a_8bit
|
||||
)
|
||||
marlin_s = marlin_permute_scales(s, size_k, size_n, group_size, is_a_8bit=is_a_8bit)
|
||||
|
||||
if input_dtype == torch.float8_e4m3fn and quant_type == scalar_types.uint4b8:
|
||||
ops.marlin_int4_fp8_preprocess(marlin_q_w, inplace=True)
|
||||
marlin_s = marlin_s * 512
|
||||
|
||||
# Create result
|
||||
res_list = [w_ref, marlin_q_w, marlin_s]
|
||||
for i in range(len(res_list)):
|
||||
res_list[i] = res_list[i].to(w.device)
|
||||
|
||||
return res_list
|
||||
|
||||
|
||||
def awq_marlin_quantize(
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
):
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
size_k, size_n = w.shape
|
||||
|
||||
# Normalize group_size
|
||||
if group_size == -1:
|
||||
group_size = size_k
|
||||
assert group_size <= size_k
|
||||
|
||||
# Detect num groups
|
||||
assert size_k % group_size == 0
|
||||
num_groups = size_k // group_size
|
||||
|
||||
# Quantize with zp
|
||||
w_ref, q_w, s, zp = quantize_weights(w, quant_type, group_size, zero_points=True)
|
||||
|
||||
if input_dtype == torch.float8_e4m3fn and quant_type == scalar_types.uint4:
|
||||
repeated_zp = zp.repeat_interleave(group_size, 0)
|
||||
q_w_old = q_w
|
||||
q_w = q_w_old - repeated_zp
|
||||
q_w[q_w < 0] = 15 - q_w_old[q_w < 0]
|
||||
s = s * 512
|
||||
|
||||
# Reformat to marlin
|
||||
weight_perm = get_weight_perm(quant_type.size_bits, is_a_8bit)
|
||||
marlin_q_w = marlin_weights(
|
||||
q_w, size_k, size_n, quant_type.size_bits, weight_perm, is_a_8bit=is_a_8bit
|
||||
)
|
||||
marlin_s = marlin_permute_scales(s, size_k, size_n, group_size, is_a_8bit=is_a_8bit)
|
||||
marlin_zp = marlin_zero_points(
|
||||
zp, num_groups, size_n, quant_type.size_bits, is_a_8bit=is_a_8bit
|
||||
)
|
||||
|
||||
# Create result
|
||||
res_list = [w_ref, marlin_q_w, marlin_s, marlin_zp]
|
||||
for i in range(len(res_list)):
|
||||
res_list[i] = res_list[i].to(w.device)
|
||||
|
||||
return res_list
|
||||
@@ -1,279 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from
|
||||
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
|
||||
# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py
|
||||
# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
|
||||
#
|
||||
# Copyright 2023 The Qwen team.
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
||||
# and OPT implementations in this library. It has been modified from its
|
||||
# original forms to accommodate minor architectural differences compared
|
||||
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Shared resampler perceiver network used in multimodal models and
|
||||
related helpers for sincos positional embeddings.
|
||||
|
||||
Example models: Qwen (Qwen-VL), MiniCPM-V 2.0
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from functools import partial
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
|
||||
DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
|
||||
|
||||
|
||||
def get_abs_pos(abs_pos: torch.Tensor, tgt_size: torch.Tensor | int) -> torch.Tensor:
|
||||
# abs_pos: L, C
|
||||
# tgt_size: (H, W)
|
||||
# return: M, C
|
||||
src_size = int(math.sqrt(abs_pos.size(0)))
|
||||
dtype = abs_pos.dtype
|
||||
if isinstance(tgt_size, int):
|
||||
tgt_size = (tgt_size, tgt_size)
|
||||
if src_size == tgt_size[0] and src_size == tgt_size[1]:
|
||||
return abs_pos
|
||||
return (
|
||||
F.interpolate(
|
||||
abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
|
||||
size=(tgt_size[0], tgt_size[1]),
|
||||
mode="bicubic",
|
||||
align_corners=False,
|
||||
)
|
||||
.permute(0, 2, 3, 1)
|
||||
.flatten(0, 2)
|
||||
.to(dtype=dtype)
|
||||
)
|
||||
|
||||
|
||||
# sin/cos positional embedding helpers are adapted from:
|
||||
# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
|
||||
def get_1d_sincos_pos_embed_from_grid(
|
||||
embed_dim: int, pos: np.ndarray, version: tuple[int, int] = (2, 0)
|
||||
) -> torch.Tensor:
|
||||
"""embed_dim: output dimension for each position
|
||||
pos: a list of positions to be encoded: size (M,) / (H, W)
|
||||
out: (M, D) / (H, W, D)
|
||||
"""
|
||||
assert embed_dim % 2 == 0
|
||||
omega = np.arange(embed_dim // 2, dtype=np.float32)
|
||||
omega /= embed_dim / 2.0
|
||||
omega = 1.0 / 10000**omega # (D/2,)
|
||||
|
||||
if version == (2, 0):
|
||||
pos = pos.reshape(-1) # (M,)
|
||||
out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product
|
||||
emb_sin = np.sin(out) # (M, D/2)
|
||||
emb_cos = np.cos(out) # (M, D/2)
|
||||
emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
|
||||
else:
|
||||
out = np.einsum("hw,d->hwd", pos, omega) # (H, W, D/2), outer product
|
||||
emb_sin = np.sin(out) # (H, W, D/2)
|
||||
emb_cos = np.cos(out) # (H, W, D/2)
|
||||
emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
|
||||
return emb
|
||||
|
||||
|
||||
def get_2d_sincos_pos_embed_from_grid(
|
||||
embed_dim: int, grid: np.ndarray, version: tuple[int, int] = (2, 0)
|
||||
) -> torch.Tensor:
|
||||
assert embed_dim % 2 == 0
|
||||
|
||||
# use half of dimensions to encode grid_h
|
||||
emb_h = get_1d_sincos_pos_embed_from_grid(
|
||||
embed_dim // 2, grid[0], version
|
||||
) # (H*W, D/2) or (H, W, D/2)
|
||||
emb_w = get_1d_sincos_pos_embed_from_grid(
|
||||
embed_dim // 2, grid[1], version
|
||||
) # (H*W, D/2) or (H, W, D/2)
|
||||
|
||||
if version == (2, 0):
|
||||
emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
|
||||
else:
|
||||
emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D)
|
||||
return emb
|
||||
|
||||
|
||||
def get_2d_sincos_pos_embed(
|
||||
embed_dim: int,
|
||||
grid_size: int | tuple[int, int],
|
||||
cls_token: bool = False,
|
||||
version: tuple[int, int] = (2, 0),
|
||||
) -> torch.Tensor:
|
||||
"""grid_size: int of the grid height and width
|
||||
return:
|
||||
pos_embed: [grid_size*grid_size, embed_dim] or
|
||||
[1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
|
||||
"""
|
||||
if isinstance(grid_size, int):
|
||||
grid_h_size, grid_w_size = grid_size, grid_size
|
||||
else:
|
||||
grid_h_size, grid_w_size = grid_size[0], grid_size[1]
|
||||
|
||||
grid_h = np.arange(grid_h_size, dtype=np.float32)
|
||||
grid_w = np.arange(grid_w_size, dtype=np.float32)
|
||||
grid = np.meshgrid(grid_w, grid_h) # here w goes first
|
||||
grid = np.stack(grid, axis=0)
|
||||
assert isinstance(grid, np.ndarray) and grid.shape == (2, grid_h_size, grid_w_size)
|
||||
|
||||
if version == (2, 0):
|
||||
grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
|
||||
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
|
||||
if cls_token:
|
||||
pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
|
||||
else:
|
||||
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
|
||||
return pos_embed
|
||||
|
||||
|
||||
class BaseResampler(nn.Module):
|
||||
"""A 2D perceiver-resampler network with one cross attention layers by
|
||||
(grid_size**2) learnable queries and 2d sincos pos_emb.
|
||||
Outputs:
|
||||
A tensor with the shape of (grid_size**2, embed_dim)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_queries: int,
|
||||
embed_dim: int,
|
||||
num_heads: int,
|
||||
kv_dim: int | None = None,
|
||||
norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
|
||||
do_post_projection: bool = True,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.num_queries = num_queries
|
||||
self.embed_dim = embed_dim
|
||||
self.num_heads = num_heads
|
||||
|
||||
self.query = nn.Parameter(torch.empty(self.num_queries, embed_dim))
|
||||
|
||||
if kv_dim is not None and kv_dim != embed_dim:
|
||||
self.kv_proj = ReplicatedLinear(
|
||||
kv_dim,
|
||||
embed_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.kv_proj",
|
||||
)
|
||||
else:
|
||||
# Maintain the same return value with ReplicatedLinear.forward
|
||||
self.kv_proj = lambda *args, **kwargs: ( # type: ignore # noqa
|
||||
nn.Identity()(*args, **kwargs),
|
||||
None,
|
||||
)
|
||||
self.attn = nn.MultiheadAttention(embed_dim, num_heads)
|
||||
self.ln_q = norm_layer(embed_dim)
|
||||
self.ln_kv = norm_layer(embed_dim)
|
||||
self.do_post_projection = do_post_projection
|
||||
if self.do_post_projection:
|
||||
self.ln_post = norm_layer(embed_dim)
|
||||
data = (embed_dim**-0.5) * torch.empty(embed_dim, embed_dim)
|
||||
self.proj = nn.Parameter(data=data)
|
||||
|
||||
def _repeat(self, query, N: int):
|
||||
return query.unsqueeze(1).repeat(1, N, 1)
|
||||
|
||||
|
||||
class Resampler2(BaseResampler):
|
||||
"""Resampler-perceiver network to be used for a variety of model types,
|
||||
e.g., Qwen-vl / Minicpmv 2.0. The main difference is the addition of the
|
||||
do_post_projection arg, which indicates whether or not there should be
|
||||
a post layer normalization and projector after the attention. This is
|
||||
present in minicpmv2.0, but not qwen-vl.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
grid_size: int,
|
||||
embed_dim: int,
|
||||
num_heads: int,
|
||||
kv_dim: int | None = None,
|
||||
norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
|
||||
adaptive: bool = False,
|
||||
do_post_projection: bool = True,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
grid_size**2,
|
||||
embed_dim,
|
||||
num_heads,
|
||||
kv_dim,
|
||||
norm_layer,
|
||||
do_post_projection=do_post_projection,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
self.adaptive = adaptive
|
||||
pos_embed_arr = get_2d_sincos_pos_embed(embed_dim, grid_size, version=(2, 0))
|
||||
|
||||
self.pos_embed = nn.Parameter(
|
||||
torch.from_numpy(pos_embed_arr).requires_grad_(False)
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
tgt_sizes: torch.Tensor | None = None,
|
||||
attn_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if tgt_sizes is None:
|
||||
tgt_sizes = int(math.sqrt(x.size(1)))
|
||||
if self.adaptive:
|
||||
pos_embed_arr = get_2d_sincos_pos_embed(
|
||||
self.embed_dim, tgt_sizes, version=(2, 0)
|
||||
)
|
||||
pos_embed = torch.from_numpy(pos_embed_arr).to(
|
||||
device=x.device, dtype=x.dtype
|
||||
)
|
||||
else:
|
||||
pos_embed = get_abs_pos(self.pos_embed, tgt_sizes).to(
|
||||
device=x.device, dtype=x.dtype
|
||||
)
|
||||
|
||||
x, _ = self.kv_proj(x)
|
||||
x = self.ln_kv(x).permute(1, 0, 2)
|
||||
|
||||
N = x.shape[1]
|
||||
q = self.ln_q(self.query)
|
||||
out = self.attn(
|
||||
self._repeat(q, N) + self.pos_embed.unsqueeze(1),
|
||||
x + pos_embed.unsqueeze(1),
|
||||
x,
|
||||
attn_mask=attn_mask,
|
||||
)[0]
|
||||
x = out.permute(1, 0, 2)
|
||||
if self.do_post_projection:
|
||||
x = self.ln_post(x)
|
||||
x = x @ self.proj
|
||||
return x
|
||||
@@ -1,93 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from .mrope import MRotaryEmbedding
|
||||
|
||||
|
||||
class Ernie4_5_VLRotaryEmbedding(MRotaryEmbedding):
|
||||
"""3D rotary positional embedding. 3D is t:time h:height w:width."""
|
||||
|
||||
def forward_native( # type: ignore[override]
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
assert positions.ndim == 1 or positions.ndim == 2
|
||||
assert key is not None
|
||||
|
||||
num_tokens = positions.shape[-1]
|
||||
cos_sin = self.cos_sin_cache[positions]
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
if positions.ndim == 2:
|
||||
assert self.mrope_section
|
||||
|
||||
section_h = self.mrope_section[0] # 22
|
||||
section_w = self.mrope_section[1] # 22
|
||||
section_t = self.mrope_section[2] # 20
|
||||
assert section_h == section_w
|
||||
# Split according to [h w h w h w h w... t t t...]
|
||||
section_cos_t = cos[..., -section_t:]
|
||||
section_cos_h = cos[..., : section_h + section_w : 2]
|
||||
section_cos_w = cos[..., 1 : section_h + section_w : 2]
|
||||
|
||||
cos_t, cos_h, cos_w = section_cos_t[0], section_cos_h[1], section_cos_w[2]
|
||||
cos_hw = torch.stack([cos_h, cos_w], dim=-1).reshape(
|
||||
cos_h.shape[:-1] + (cos_h.shape[-1] * 2,)
|
||||
)
|
||||
cos = torch.cat([cos_hw, cos_t], dim=-1)
|
||||
|
||||
section_sin_t = sin[..., -section_t:]
|
||||
section_sin_h = sin[..., : section_h + section_w : 2]
|
||||
section_sin_w = sin[..., 1 : section_h + section_w : 2]
|
||||
|
||||
sin_t, sin_h, sin_w = section_sin_t[0], section_sin_h[1], section_sin_w[2]
|
||||
sin_hw = torch.stack([sin_h, sin_w], dim=-1).reshape(
|
||||
sin_h.shape[:-1] + (sin_h.shape[-1] * 2,)
|
||||
)
|
||||
sin = torch.cat([sin_hw, sin_t], dim=-1)
|
||||
|
||||
query_shape = query.shape
|
||||
query = query.view(num_tokens, -1, self.head_size)
|
||||
query_rot = query[..., : self.rotary_dim]
|
||||
query_pass = query[..., self.rotary_dim :]
|
||||
query_rot = self.apply_rotary_emb.forward_native(
|
||||
query_rot,
|
||||
cos,
|
||||
sin,
|
||||
)
|
||||
query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
|
||||
|
||||
key_shape = key.shape
|
||||
key = key.view(num_tokens, -1, self.head_size)
|
||||
key_rot = key[..., : self.rotary_dim]
|
||||
key_pass = key[..., self.rotary_dim :]
|
||||
key_rot = self.apply_rotary_emb.forward_native(
|
||||
key_rot,
|
||||
cos,
|
||||
sin,
|
||||
)
|
||||
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
|
||||
return query, key
|
||||
|
||||
def forward_cuda( # type: ignore[override]
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
return self.forward_native(positions, query, key)
|
||||
|
||||
def forward_xpu( # type: ignore[override]
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
# No fused XPU kernel for this 3D t/h/w rope; base
|
||||
# MRotaryEmbedding.forward_xpu forwards an extra `offsets` arg that
|
||||
# this class's forward_cuda override doesn't accept. Use native path.
|
||||
return self.forward_native(positions, query, key)
|
||||
@@ -1,153 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
|
||||
# num_tokens/num_grids vary per request (incl. the ==1 class) and grids_ptr
|
||||
# is an offset slice of the packed H2D buffer, so its 16B alignment varies
|
||||
# with grid count; without these the kernel recompiles mid-serving.
|
||||
@triton.jit(
|
||||
do_not_specialize=["num_tokens", "num_grids"],
|
||||
do_not_specialize_on_alignment=["grids_ptr"],
|
||||
)
|
||||
def _vit_mrope_setup_kernel(
|
||||
inv_freq_t_ptr,
|
||||
inv_freq_h_ptr,
|
||||
inv_freq_w_ptr,
|
||||
cos_out_ptr,
|
||||
sin_out_ptr,
|
||||
cu_seqlens_ptr, # (G + 1,) int32 token boundaries per grid
|
||||
grids_ptr, # (G, 3) int32: t, h, w per grid
|
||||
num_tokens,
|
||||
num_grids,
|
||||
half_t: tl.constexpr,
|
||||
half_h: tl.constexpr,
|
||||
half_w: tl.constexpr,
|
||||
spatial_merge: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""Build triton_mrope's (3, N, half_rot) cos/sin t/h/w planes in one
|
||||
launch: compute each token's (t, h, w) position arithmetically and the
|
||||
rotation in fp32 from the per-axis inv_freq. Only each axis's mrope
|
||||
section columns are written; the rest is left uninitialized (the
|
||||
consumer's per-axis masked loads read only the section columns)."""
|
||||
pid = tl.program_id(0)
|
||||
tok = pid * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
tok_mask = tok < num_tokens
|
||||
|
||||
# Grid (segment) index per token; cu_seqlens is nondecreasing.
|
||||
seg = tl.zeros([BLOCK_N], dtype=tl.int32)
|
||||
for g in range(num_grids):
|
||||
end = tl.load(cu_seqlens_ptr + g + 1)
|
||||
seg += (tok >= end).to(tl.int32)
|
||||
|
||||
seg_start = tl.load(cu_seqlens_ptr + seg, mask=tok_mask, other=0)
|
||||
grid_h = tl.load(grids_ptr + seg * 3 + 1, mask=tok_mask, other=1)
|
||||
grid_w = tl.load(grids_ptr + seg * 3 + 2, mask=tok_mask, other=1)
|
||||
|
||||
local = tok - seg_start
|
||||
rem = local % (grid_h * grid_w)
|
||||
tpos = local // (grid_h * grid_w)
|
||||
# Within a frame tokens are ordered by spatial-merge block (bh, bw, mh,
|
||||
# mw), matching the reference ViT position-id permute.
|
||||
bh = rem // (spatial_merge * grid_w)
|
||||
r = rem % (spatial_merge * grid_w)
|
||||
bw = r // (spatial_merge * spatial_merge)
|
||||
r2 = r % (spatial_merge * spatial_merge)
|
||||
hpos = bh * spatial_merge + r2 // spatial_merge
|
||||
wpos = bw * spatial_merge + r2 % spatial_merge
|
||||
|
||||
d = tl.arange(0, BLOCK_D)
|
||||
half_rot = half_t + half_h + half_w
|
||||
row = (tok * half_rot)[:, None] + d[None, :]
|
||||
plane = num_tokens * half_rot
|
||||
|
||||
# Per-axis compute+store to cap live registers; the fp32 trig sequence
|
||||
# wants few elements per thread (hence small BLOCK_N, many warps).
|
||||
in_t = d < half_t
|
||||
inv_t = tl.load(inv_freq_t_ptr + d, mask=in_t, other=0.0)
|
||||
freq = tpos.to(tl.float32)[:, None] * inv_t[None, :]
|
||||
mask = tok_mask[:, None] & in_t[None, :]
|
||||
tl.store(cos_out_ptr + row, tl.cos(freq), mask=mask)
|
||||
tl.store(sin_out_ptr + row, tl.sin(freq), mask=mask)
|
||||
|
||||
in_h = (d >= half_t) & (d < half_t + half_h)
|
||||
inv_h = tl.load(inv_freq_h_ptr + d - half_t, mask=in_h, other=0.0)
|
||||
freq = hpos.to(tl.float32)[:, None] * inv_h[None, :]
|
||||
mask = tok_mask[:, None] & in_h[None, :]
|
||||
tl.store(cos_out_ptr + plane + row, tl.cos(freq), mask=mask)
|
||||
tl.store(sin_out_ptr + plane + row, tl.sin(freq), mask=mask)
|
||||
|
||||
in_w = (d >= half_t + half_h) & (d < half_rot)
|
||||
inv_w = tl.load(inv_freq_w_ptr + d - half_t - half_h, mask=in_w, other=0.0)
|
||||
freq = wpos.to(tl.float32)[:, None] * inv_w[None, :]
|
||||
mask = tok_mask[:, None] & in_w[None, :]
|
||||
tl.store(cos_out_ptr + 2 * plane + row, tl.cos(freq), mask=mask)
|
||||
tl.store(sin_out_ptr + 2 * plane + row, tl.sin(freq), mask=mask)
|
||||
|
||||
|
||||
def vit_mrope_setup(
|
||||
inv_freq_t: torch.Tensor,
|
||||
inv_freq_h: torch.Tensor,
|
||||
inv_freq_w: torch.Tensor,
|
||||
grid_thw: list[list[int]],
|
||||
spatial_merge_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""(3, total_N, half_rot_dim) fp32 cos/sin t/h/w planes for triton_mrope.
|
||||
|
||||
One fused kernel: positions are computed arithmetically from the grids
|
||||
and the rotation comes from the per-axis inv_freq, so there is no
|
||||
cos/sin cache to size or grow.
|
||||
|
||||
Args:
|
||||
inv_freq_t: t-axis inverse frequencies, (t_dim // 2,).
|
||||
inv_freq_h: h-axis inverse frequencies, (h_dim // 2,).
|
||||
inv_freq_w: w-axis inverse frequencies, (w_dim // 2,).
|
||||
grid_thw: per-grid [t, h, w] patch counts (host metadata).
|
||||
spatial_merge_size: spatial merge factor of the ViT.
|
||||
|
||||
"""
|
||||
device = inv_freq_t.device
|
||||
cu = [0]
|
||||
for t, h, w in grid_thw:
|
||||
cu.append(cu[-1] + t * h * w)
|
||||
n = cu[-1]
|
||||
g = len(grid_thw)
|
||||
# grid_thw is per-request host metadata, so one H2D is unavoidable;
|
||||
# pack cu_seqlens and grids into a single pinned async copy.
|
||||
buf = async_tensor_h2d(
|
||||
cu + [d for grid in grid_thw for d in grid], device=device, dtype=torch.int32
|
||||
)
|
||||
cu_seqlens = buf[: g + 1]
|
||||
grids = buf[g + 1 :].view(g, 3)
|
||||
|
||||
half_t = inv_freq_t.numel()
|
||||
half_h = inv_freq_h.numel()
|
||||
half_w = inv_freq_w.numel()
|
||||
half_rot = half_t + half_h + half_w
|
||||
cos = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
|
||||
sin = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
|
||||
_vit_mrope_setup_kernel[(triton.cdiv(n, 32),)](
|
||||
inv_freq_t,
|
||||
inv_freq_h,
|
||||
inv_freq_w,
|
||||
cos,
|
||||
sin,
|
||||
cu_seqlens,
|
||||
grids,
|
||||
n,
|
||||
g,
|
||||
half_t=half_t,
|
||||
half_h=half_h,
|
||||
half_w=half_w,
|
||||
spatial_merge=spatial_merge_size,
|
||||
BLOCK_N=32,
|
||||
BLOCK_D=triton.next_power_of_2(half_rot),
|
||||
num_warps=8,
|
||||
)
|
||||
return cos, sin
|
||||
@@ -1,83 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""In-place interleaved RoPE on the Q/K slices of a packed QKV buffer."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
# seqlen is not specialized: image sizes vary per request.
|
||||
@triton.jit(do_not_specialize=["seqlen"])
|
||||
def _packed_qk_rope_kernel(
|
||||
qk_ptr, # (seqlen, 2 * nheads, headdim) strided view of the packed QKV
|
||||
freqs_ptr, # (seqlen, rotary_dim) fp32 re/im view of the complex freqs
|
||||
seqlen,
|
||||
qk_row_stride,
|
||||
qk_head_stride,
|
||||
rotary_dim: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
):
|
||||
BLOCK_K: tl.constexpr = triton.next_power_of_2(rotary_dim)
|
||||
pid_m = tl.program_id(0)
|
||||
pid_h = tl.program_id(1)
|
||||
rm = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
rk = tl.arange(0, BLOCK_K)
|
||||
mask_m = rm < seqlen
|
||||
|
||||
base = qk_ptr + pid_h * qk_head_stride + rm[:, None] * qk_row_stride
|
||||
mask_x = mask_m[:, None] & (rk < rotary_dim)[None, :]
|
||||
x = tl.load(base + rk[None, :], mask=mask_x, other=0.0).to(tl.float32)
|
||||
f = tl.load(freqs_ptr + rm[:, None] * rotary_dim + rk[None, :], mask=mask_x)
|
||||
cos, sin = tl.split(tl.reshape(f, (BLOCK_M, BLOCK_K // 2, 2)))
|
||||
|
||||
x0, x1 = tl.split(tl.reshape(x, (BLOCK_M, BLOCK_K // 2, 2)))
|
||||
o0 = x0 * cos - x1 * sin
|
||||
o1 = x1 * cos + x0 * sin
|
||||
tl.store(base + rk[None, :], tl.interleave(o0, o1), mask=mask_x)
|
||||
|
||||
|
||||
def packed_qk_rope_(xqkv: torch.Tensor, freqs_cis: torch.Tensor) -> None:
|
||||
"""Rotate Q and K in place inside a packed QKV buffer.
|
||||
|
||||
Args:
|
||||
xqkv: contiguous (seqlen, 3, nheads, headdim); only the Q and K
|
||||
slices are rotated.
|
||||
freqs_cis: contiguous (seqlen, headdim // 2) complex64 rotary freqs,
|
||||
read through its interleaved re/im fp32 view.
|
||||
|
||||
Matches ``ApplyRotaryEmb(enable_fp32_compute=True)`` applied to Q and K
|
||||
separately to within one ULP, but in one kernel with ~5x less memory
|
||||
traffic. Both compute in fp32; the gap is which product the compiler
|
||||
keeps exact inside the fused multiply-add, which differs per backend.
|
||||
Requires triton: callers must check HAS_TRITON and use the unfused path
|
||||
otherwise. Contract violations raise.
|
||||
|
||||
"""
|
||||
assert xqkv.ndim == 4 and xqkv.size(1) == 3, xqkv.shape
|
||||
assert xqkv.is_contiguous(), xqkv.shape
|
||||
assert freqs_cis.ndim == 2 and freqs_cis.is_contiguous(), freqs_cis.shape
|
||||
assert freqs_cis.dtype == torch.complex64, freqs_cis.dtype
|
||||
|
||||
seq_length, _, num_heads, head_dim = xqkv.shape
|
||||
rotary_dim = freqs_cis.size(-1) * 2
|
||||
assert freqs_cis.size(0) == seq_length, (freqs_cis.shape, xqkv.shape)
|
||||
assert rotary_dim == head_dim, (freqs_cis.shape, xqkv.shape) # full rotation
|
||||
qk = xqkv.as_strided(
|
||||
(seq_length, 2 * num_heads, head_dim),
|
||||
(3 * num_heads * head_dim, head_dim, 1),
|
||||
)
|
||||
|
||||
BLOCK_M = 4
|
||||
# seqlen on grid.x: HIP caps gridDim.y at 65535.
|
||||
grid = (triton.cdiv(seq_length, BLOCK_M), 2 * num_heads)
|
||||
_packed_qk_rope_kernel[grid](
|
||||
qk,
|
||||
freqs_cis.view(torch.float32),
|
||||
seq_length,
|
||||
qk.stride(0),
|
||||
qk.stride(1),
|
||||
rotary_dim=rotary_dim,
|
||||
BLOCK_M=BLOCK_M,
|
||||
num_warps=2 if rotary_dim <= 64 else 4,
|
||||
)
|
||||
@@ -23,7 +23,6 @@ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
|
||||
|
||||
from vllm.model_executor.models.transformers.base import VLLM_ATTN_ATTR
|
||||
from vllm.model_executor.models.transformers.causal import CausalMixin
|
||||
from vllm.model_executor.models.transformers.legacy import LegacyMixin
|
||||
from vllm.model_executor.models.transformers.moe import MoEMixin
|
||||
from vllm.model_executor.models.transformers.multimodal import (
|
||||
MultiModalDummyInputsBuilder,
|
||||
@@ -31,10 +30,6 @@ from vllm.model_executor.models.transformers.multimodal import (
|
||||
MultiModalProcessingInfo,
|
||||
MultiModalProcessor,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.pooling import (
|
||||
EmbeddingMixin,
|
||||
SequenceClassificationMixin,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -149,42 +144,6 @@ class TransformersMultiModalForCausalLM(MultiModalMixin, CausalMixin): ...
|
||||
class TransformersMultiModalMoEForCausalLM(MoEMixin, MultiModalMixin, CausalMixin): ...
|
||||
|
||||
|
||||
# Embedding models
|
||||
class TransformersEmbeddingModel(EmbeddingMixin, LegacyMixin): ...
|
||||
|
||||
|
||||
class TransformersMoEEmbeddingModel(EmbeddingMixin, MoEMixin): ...
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
MultiModalProcessor,
|
||||
info=MultiModalProcessingInfo,
|
||||
dummy_inputs=MultiModalDummyInputsBuilder,
|
||||
)
|
||||
class TransformersMultiModalEmbeddingModel(EmbeddingMixin, MultiModalMixin): ...
|
||||
|
||||
|
||||
# Sequence classification models
|
||||
class TransformersForSequenceClassification(
|
||||
SequenceClassificationMixin, LegacyMixin
|
||||
): ...
|
||||
|
||||
|
||||
class TransformersMoEForSequenceClassification(
|
||||
SequenceClassificationMixin, MoEMixin
|
||||
): ...
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
MultiModalProcessor,
|
||||
info=MultiModalProcessingInfo,
|
||||
dummy_inputs=MultiModalDummyInputsBuilder,
|
||||
)
|
||||
class TransformersMultiModalForSequenceClassification(
|
||||
SequenceClassificationMixin, MultiModalMixin
|
||||
): ...
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Handle imports of non-existent classes with a helpful error message."""
|
||||
if name not in globals():
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend mixins for pooling models."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import AutoModelForSequenceClassification
|
||||
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.model_executor.layers.pooler import DispatchPooler
|
||||
from vllm.model_executor.models.interfaces import SupportsCrossEncoding
|
||||
from vllm.model_executor.models.interfaces_base import VllmModelForPooling
|
||||
|
||||
from .base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
class ClassifierWithReshape(nn.Module):
|
||||
"""Token extraction has already been applied in `pooler.pooling`.
|
||||
|
||||
Add dim to match expected input shape of `classifier.forward`.
|
||||
"""
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
if len(args) > 0:
|
||||
args = (args[0].unsqueeze(1), *args[1:])
|
||||
return super().forward(*args, **kwargs)
|
||||
|
||||
|
||||
class EmbeddingMixin(VllmModelForPooling, Base):
|
||||
default_seq_pooling_type = "CLS"
|
||||
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
# Skip VllmModelForPooling.__init__ and call the next class in MRO
|
||||
super(VllmModelForPooling, self).__init__(
|
||||
vllm_config=vllm_config, prefix=prefix
|
||||
)
|
||||
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
self.pooler = DispatchPooler.for_embedding(pooler_config)
|
||||
|
||||
|
||||
class SequenceClassificationMixin(SupportsCrossEncoding, VllmModelForPooling, Base):
|
||||
default_seq_pooling_type = "CLS"
|
||||
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
# Skip VllmModelForPooling.__init__ and call the next class in MRO
|
||||
super(VllmModelForPooling, self).__init__(
|
||||
vllm_config=vllm_config, prefix=prefix
|
||||
)
|
||||
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
# Certain information about the model and classifier can only be
|
||||
# inferred from the `ForSequenceClassification` class. Therefore, we
|
||||
# instantiate it on the "meta" device to avoid allocating GPU memory.
|
||||
with torch.device("meta"):
|
||||
seq_cls_model = AutoModelForSequenceClassification.from_config(
|
||||
self.config,
|
||||
dtype=self.model_config.dtype,
|
||||
trust_remote_code=self.model_config.trust_remote_code,
|
||||
)
|
||||
|
||||
# When used for sequence classification, some models have their
|
||||
# pooling layers removed. Make sure this is reflected in vLLM.
|
||||
for module in seq_cls_model.modules():
|
||||
if hasattr(module, "pooler") and module.pooler is None:
|
||||
self.model.pooler = None
|
||||
break
|
||||
|
||||
# Unlike `lm_head`, `classifier` is not always `nn.Linear`.
|
||||
self.classifier = getattr_iter(seq_cls_model, ["classifier", "score"], None)
|
||||
if self.classifier is None:
|
||||
raise ValueError(
|
||||
"Could not find `classifier` or `score` layer in the "
|
||||
"`AutoModelForSequenceClassification` instance."
|
||||
)
|
||||
self.init_parameters(self.classifier, dtype=self.model_config.head_dtype)
|
||||
|
||||
# Order `ClassifierWithReshape` ahead of the classifier's own class so that
|
||||
# its `super().forward(...)` reaches the original implementation.
|
||||
self.classifier.__class__ = type(
|
||||
"ClassifierWithReshape",
|
||||
(ClassifierWithReshape, type(self.classifier)),
|
||||
{},
|
||||
)
|
||||
|
||||
self.pooler = DispatchPooler.for_seq_cls(
|
||||
pooler_config,
|
||||
classifier=self.classifier,
|
||||
)
|
||||
Reference in New Issue
Block a user