[Mini-VLLM] Remove the ROCm device path, debug routes and arch-gated tests
- Delete csrc/rocm, hipify, the _rocm_C extension and the setup.py rocm target path; drop the rocm requirement files - Delete the aiter, rdna3, skinny-gemm and ROCm fused-MoE kernel modules and retarget the linear kernel registries away from them - Reduce platforms/rocm.py to the GCN architecture predicates that live code still imports, and delete the RocmPlatform class and plugin - Drop the VLLM_SERVER_DEV_MODE debug surface: reset, collective_rpc, server_info and sleep routes plus the log_error_stack flag - Delete lm-format-enforcer, lark and otel-semantic-conventions-ai pins; the live server process maps none of them - Delete tests that only build architectures this fork no longer registers, and test directories left without any test file - Restore the tests/v1/kv_connector mock helpers that scheduler tests import at module level Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -50,12 +50,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.machete import (
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.marlin import (
|
||||
MarlinLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import (
|
||||
RDNA3W4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import (
|
||||
RDNAHybridW4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
|
||||
TritonW4A16LinearKernel,
|
||||
)
|
||||
@@ -122,9 +116,6 @@ from vllm.model_executor.kernels.linear.mxfp8.marlin import (
|
||||
from vllm.model_executor.kernels.linear.mxfp8.pytorch import (
|
||||
TorchMxFp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.nvfp4 import (
|
||||
NvFp4LinearKernel,
|
||||
NvFp4LinearLayerConfig,
|
||||
@@ -210,9 +201,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.pytorch import (
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
RowWiseTorchFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.rocm import (
|
||||
ROCmFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
@@ -439,7 +427,6 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
|
||||
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
|
||||
AiterPreshuffledPerTokenFp8ScaledMMLinearKernel,
|
||||
AiterPerTokenFp8ScaledMMLinearKernel,
|
||||
ROCmFP8ScaledMMLinearKernel,
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
RowWiseTorchFP8ScaledMMLinearKernel,
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
@@ -502,8 +489,6 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
|
||||
HummingLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
RDNA3W4A16LinearKernel,
|
||||
RDNAHybridW4A16LinearKernel,
|
||||
TritonW4A16LinearKernel,
|
||||
ConchLinearKernel,
|
||||
ExllamaLinearKernel,
|
||||
@@ -528,9 +513,6 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
|
||||
TorchMxFp8LinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
|
||||
# falls through to BF16 emulation (hipBLASLt) elsewhere / on regression.
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
EmulationMxfp8LinearKernel,
|
||||
],
|
||||
}
|
||||
@@ -1283,7 +1265,6 @@ __all__ = [
|
||||
"ChannelWiseTorchFP8ScaledMMLinearKernel",
|
||||
"PerTensorTorchFP8ScaledMMLinearKernel",
|
||||
"RowWiseTorchFP8ScaledMMLinearKernel",
|
||||
"ROCmFP8ScaledMMLinearKernel",
|
||||
"TritonInt8ScaledMMLinearKernel",
|
||||
"ZentorchInt8ScaledMMLinearKernel",
|
||||
"ZentorchWNA16LinearKernel",
|
||||
@@ -1294,7 +1275,6 @@ __all__ = [
|
||||
"CutlassW4A8LinearKernel",
|
||||
"Dynamic4bitLinearKernel",
|
||||
"ExllamaLinearKernel",
|
||||
"RDNAHybridW4A16LinearKernel",
|
||||
"MacheteLinearKernel",
|
||||
"MarlinLinearKernel",
|
||||
"TritonW4A16LinearKernel",
|
||||
|
||||
@@ -29,12 +29,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearKernel,
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import (
|
||||
RDNA3W4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import (
|
||||
RDNAHybridW4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
|
||||
TritonW4A16LinearKernel,
|
||||
)
|
||||
@@ -53,8 +47,6 @@ __all__ = [
|
||||
"HummingLinearKernel",
|
||||
"MacheteLinearKernel",
|
||||
"MarlinLinearKernel",
|
||||
"RDNA3W4A16LinearKernel",
|
||||
"RDNAHybridW4A16LinearKernel",
|
||||
"TritonW4A16LinearKernel",
|
||||
"XPUwNa16LinearKernel",
|
||||
"ZentorchWNA16LinearKernel",
|
||||
|
||||
@@ -1,163 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""W4A16 GPTQ kernel for AMD RDNA3 (gfx1100) — fp16 + bf16.
|
||||
|
||||
Drop-in replacement for ExllamaLinearKernel on RDNA3 that adds native bf16
|
||||
support. The HIP kernel lives in ``csrc/rocm/q_gemm_rdna3.cu``
|
||||
and is exposed via ``torch.ops._rocm_C.gptq_gemm_rdna3``.
|
||||
|
||||
Registered ahead of TritonW4A16LinearKernel for the ROCm-RDNA3 path; falls
|
||||
through to the Triton kernel on non-RDNA3 ROCm devices (e.g. CDNA/MI300).
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
pack_quantized_values_into_int32,
|
||||
)
|
||||
from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
|
||||
|
||||
|
||||
class RDNA3W4A16LinearKernel(MPLinearKernel):
|
||||
SUPPORTED_QUANT_TYPES = [scalar_types.uint4b8]
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
# ROCm gates via on_gfx1100() in can_implement.
|
||||
return 60
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_rocm():
|
||||
return False, "RDNA3 W4A16 kernel is ROCm-only"
|
||||
|
||||
from vllm.platforms.rocm import on_gfx1100
|
||||
|
||||
if not on_gfx1100():
|
||||
return False, "RDNA3 W4A16 kernel requires gfx1100"
|
||||
|
||||
# The HIP op is registered by the C++ extension; if a user is running
|
||||
# against a vLLM build that doesn't include it (e.g. partial rebuild),
|
||||
# fall through gracefully to the next kernel in the registry.
|
||||
if not (
|
||||
hasattr(torch.ops, "_rocm_C")
|
||||
and hasattr(torch.ops._rocm_C, "gptq_gemm_rdna3")
|
||||
):
|
||||
return (
|
||||
False,
|
||||
"torch.ops._rocm_C.gptq_gemm_rdna3 missing — rebuild C++ extension",
|
||||
)
|
||||
|
||||
if c.act_type not in (torch.float16, torch.bfloat16):
|
||||
return False, "RDNA3 W4A16 kernel only supports fp16 and bf16"
|
||||
|
||||
if c.weight_type not in cls.SUPPORTED_QUANT_TYPES:
|
||||
return (
|
||||
False,
|
||||
f"Quant type ({c.weight_type}) not supported by "
|
||||
f"RDNA3 W4A16 kernel; supported: {cls.SUPPORTED_QUANT_TYPES}",
|
||||
)
|
||||
|
||||
if c.group_size <= 0:
|
||||
return (
|
||||
False,
|
||||
"RDNA3 W4A16 kernel does not support channelwise quantization",
|
||||
)
|
||||
|
||||
if c.full_weight_shape[0] % c.group_size != 0:
|
||||
return (
|
||||
False,
|
||||
f"Group size ({c.group_size}) does not evenly divide K "
|
||||
f"({c.full_weight_shape[0]})",
|
||||
)
|
||||
|
||||
# Output features must be a multiple of the pack factor (8 nibbles per
|
||||
# int32) and of 8 so that qzeros (packed 4-bit per col) align cleanly
|
||||
# against the BLOCK_KN_SIZE*4 = 512 N-stride and per-thread 4 columns.
|
||||
if c.partition_weight_shape[1] % 8 != 0:
|
||||
return (
|
||||
False,
|
||||
"Output features must be a multiple of 8 for the RDNA3 "
|
||||
"W4A16 kernel (qzeros packing)",
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
# ----- Weight prep (identical layout/shuffle as ExllamaLinearKernel) -----
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
c = self.config
|
||||
device = getattr(layer, self.w_q_name).device
|
||||
|
||||
# Synthesize zero points if the checkpoint doesn't carry them.
|
||||
if not c.zero_points:
|
||||
self.w_zp_name = "qzeros"
|
||||
groups = c.partition_weight_shape[0] // c.group_size
|
||||
out_features = c.partition_weight_shape[1]
|
||||
|
||||
if c.weight_type.has_bias():
|
||||
# GPTQv1 quirk: the kernel adds 1 to the stored zero, so we
|
||||
# encode (bias - 1) here. See exllama.py for the link to the
|
||||
# documentation of this checkpoint-format wart.
|
||||
zeros = torch.full(
|
||||
(groups, out_features),
|
||||
c.weight_type.bias - 1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"RDNA3 W4A16 kernel: zero-bias 4-bit quant requires "
|
||||
"explicit zero points (GPTQv1 +1 quirk)."
|
||||
)
|
||||
zeros = pack_quantized_values_into_int32(zeros, c.weight_type, packed_dim=1)
|
||||
setattr(
|
||||
layer, self.w_zp_name, torch.nn.Parameter(zeros, requires_grad=False)
|
||||
)
|
||||
|
||||
def transform_w_q(x):
|
||||
assert isinstance(x, BasevLLMParameter)
|
||||
permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
|
||||
x_cont = x.data.contiguous()
|
||||
# Same 4-bit shuffle as exllama. The RDNA3 kernel reads weights in
|
||||
# the same shuffled int32 layout and uses the (qa & 0x000F000F)
|
||||
# bit-trick on top.
|
||||
ops.gptq_shuffle(x_cont, c.weight_type.size_bits)
|
||||
return x_cont
|
||||
|
||||
def transform_w_s(x):
|
||||
assert isinstance(x, BasevLLMParameter)
|
||||
permute_param_layout_(x, input_dim=0, output_dim=1)
|
||||
x.data = x.data.contiguous()
|
||||
# Keep scales in the activation dtype (fp16 OR bf16) — the kernel
|
||||
# branches on dtype internally.
|
||||
return x.to(dtype=c.act_type)
|
||||
|
||||
self._transform_param(layer, self.w_q_name, transform_w_q)
|
||||
self._transform_param(layer, self.w_s_name, transform_w_s)
|
||||
|
||||
# ----- Forward --------------------------------------------------------
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
c = self.config
|
||||
|
||||
x_2d = x.reshape(-1, x.shape[-1])
|
||||
out_shape = x.shape[:-1] + (c.partition_weight_shape[1],)
|
||||
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
assert w_zp is not None, "Zero points are required by RDNA3 W4A16"
|
||||
output = ops.gptq_gemm_rdna3(x_2d, w_q, w_zp, w_s, False)
|
||||
|
||||
if bias is not None:
|
||||
output.add_(bias)
|
||||
return output.reshape(out_shape)
|
||||
@@ -1,651 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Hybrid W4A16 kernel: Triton for prefill, HIP skinny for decode.
|
||||
|
||||
Routes based on the activation shape:
|
||||
M <= MAX_SKINNY_BATCH_SIZE and K*M <= MEDIUM_SKINNY_LIMIT_ELEMENTS:
|
||||
HIP skinny GEMM (wvSplitK_int4_g)
|
||||
otherwise: Triton W4A16 fused dequant GEMM
|
||||
|
||||
Stores the weights ONCE as int8 [N, K//2] (ExLlama shuffle packed). Both
|
||||
paths read this single buffer: the HIP skinny kernel uses it directly, and
|
||||
the triton kernel reinterprets it as int32 [N, K//8] via a view (and
|
||||
transposes tiles in-register). No dual weight storage.
|
||||
"""
|
||||
|
||||
from contextlib import nullcontext
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
unpack_quantized_values_into_int32,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
permute_param_layout_,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
|
||||
|
||||
SUPPORTED_GROUP_SIZES = [32, 64, 128]
|
||||
|
||||
|
||||
def _on_gfx12x() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx12x
|
||||
|
||||
return on_gfx12x()
|
||||
|
||||
|
||||
def _on_gfx1x() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx1x
|
||||
|
||||
return on_gfx1x()
|
||||
|
||||
|
||||
def _on_gfx1151() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx1151
|
||||
|
||||
return on_gfx1151()
|
||||
|
||||
|
||||
def _on_gfx115x() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx115x
|
||||
|
||||
return on_gfx115x()
|
||||
|
||||
|
||||
# Maximum activation rows supported by HIP skinny (Python M maps to C++ N_in).
|
||||
# When K*M exceeds regular LDS capacity, the C++ medium kernel caches the
|
||||
# prefix in LDS and reads the remaining activation elements from global memory.
|
||||
# On gfx115x the K*M term is dropped entirely: deep-K shapes stage every
|
||||
# activation row in LDS one K chunk at a time, and only when N is not a
|
||||
# multiple of the kernel's row tile does the medium kernel fall back to reading
|
||||
# the rows that do not fit from global memory.
|
||||
MAX_SKINNY_BATCH_SIZE = 5
|
||||
# 64 KiB per-workgroup LDS limit expressed in fp16 elements.
|
||||
# (AMD RDNA has 128 KiB total LDS per CU, but 64 KiB per workgroup.)
|
||||
LDS_CAPACITY_ELEMENTS = 64 * 1024 // 2 # 32768 fp16 elements
|
||||
MEDIUM_SKINNY_LIMIT_ELEMENTS = int(LDS_CAPACITY_ELEMENTS * 1.2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Triton kernel for the prefill path (reads skinny-format weights [N, K//8])
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _triton_w4a16_skinny_fmt_kernel(
|
||||
# Pointers
|
||||
a_ptr, # [M, K] fp16/bf16 activations
|
||||
b_ptr, # [N, K//8] int32 packed (ExLlama shuffle, K is packed dim)
|
||||
scales_ptr, # [N, K//G] fp16/bf16 scales (skinny layout)
|
||||
zp_ptr, # [N//8, K//G] int32 zero-points (when HAS_ZP=True)
|
||||
c_ptr, # [M, N] fp16/bf16 output
|
||||
# Dimensions
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
K8, # K // 8
|
||||
num_groups, # K // group_size
|
||||
stride_sn, # row stride of scales_ptr and zp_ptr, in groups
|
||||
stride_bn, # b_ptr row stride; >= K8, the rows may be padded
|
||||
stride_am, # a_ptr row stride in elements; >= K, rows may be padded
|
||||
# Quantization parameters
|
||||
group_size,
|
||||
ZP_BIAS: tl.constexpr,
|
||||
HAS_ZP: tl.constexpr,
|
||||
# Block sizes
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
"""Fused W4A16 GEMM reading weights from skinny format [N, K//8].
|
||||
|
||||
B is stored as [N, K//8] int32 using ExLlama shuffle packing:
|
||||
each int32 packs 8 K-values with interleave [0,2,4,6,1,3,5,7]:
|
||||
packed = val[0] | (val[2]<<4) | (val[4]<<8) | (val[6]<<12)
|
||||
| (val[1]<<16) | (val[3]<<20) | (val[5]<<24) | (val[7]<<28)
|
||||
|
||||
Scales are [N, K//G] (skinny layout, NOT transposed).
|
||||
When HAS_ZP=True, zp_ptr holds [N//8, K//G] int32 with row n's raw
|
||||
zero-point at word[n//8] bits 4*(n%8), and dequant is
|
||||
(nibble - zp_raw) * scale.
|
||||
When HAS_ZP=False, only the constant ZP_BIAS is subtracted (symmetric).
|
||||
"""
|
||||
pid_m = tl.program_id(0)
|
||||
pid_n = tl.program_id(1)
|
||||
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
|
||||
# ExLlama unshuffle shifts: shift[j] = (j//2)*4 + (j%2)*16
|
||||
# For 8 values: [0, 16, 4, 20, 8, 24, 12, 28]
|
||||
exllama_shifts_row = (tl.arange(0, 8) // 2) * 4 + (tl.arange(0, 8) % 2) * 16
|
||||
shifts_1d = tl.reshape(
|
||||
tl.broadcast_to(exllama_shifts_row[None, :], (BLOCK_K // 8, 8)),
|
||||
(BLOCK_K,),
|
||||
)
|
||||
shifts_full = tl.broadcast_to(shifts_1d[None, :], (BLOCK_N, BLOCK_K))
|
||||
|
||||
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
|
||||
for k_start in range(0, tl.cdiv(K, BLOCK_K)):
|
||||
offs_k = k_start * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||
mask_k = offs_k < K
|
||||
|
||||
a_ptrs = a_ptr + offs_m[:, None] * stride_am + offs_k[None, :]
|
||||
mask_a = (offs_m[:, None] < M) & mask_k[None, :]
|
||||
a = tl.load(a_ptrs, mask=mask_a, other=0.0)
|
||||
|
||||
offs_k8 = k_start * (BLOCK_K // 8) + tl.arange(0, BLOCK_K // 8)
|
||||
b_ptrs = b_ptr + offs_n[:, None] * stride_bn + offs_k8[None, :]
|
||||
mask_b = (offs_n[:, None] < N) & (offs_k8[None, :] < K8)
|
||||
b_packed = tl.load(b_ptrs, mask=mask_b, other=0)
|
||||
|
||||
b = tl.interleave(b_packed, b_packed)
|
||||
b = tl.interleave(b, b)
|
||||
b = tl.interleave(b, b)
|
||||
b = (b >> shifts_full) & 0xF
|
||||
|
||||
group_idx = (k_start * BLOCK_K) // group_size
|
||||
scale_ptrs = scales_ptr + offs_n * stride_sn + group_idx
|
||||
scale_mask = offs_n < N
|
||||
scales = tl.load(scale_ptrs, mask=scale_mask, other=1.0)
|
||||
|
||||
if HAS_ZP:
|
||||
zp_ptrs = zp_ptr + (offs_n // 8) * stride_sn + group_idx
|
||||
zp_word = tl.load(zp_ptrs, mask=scale_mask, other=0)
|
||||
zp_raw = (zp_word >> (4 * (offs_n % 8))) & 0xF
|
||||
b_fp = (b - zp_raw[:, None]).to(scales.dtype) * scales[:, None]
|
||||
else:
|
||||
b_fp = (b - ZP_BIAS).to(scales.dtype) * scales[:, None]
|
||||
|
||||
b_fp_t = tl.trans(b_fp)
|
||||
accumulator += tl.dot(a, b_fp_t, out_dtype=tl.float32)
|
||||
|
||||
c = accumulator.to(c_ptr.type.element_ty)
|
||||
c_ptrs = c_ptr + offs_m[:, None] * N + offs_n[None, :]
|
||||
mask_c = (offs_m[:, None] < M) & (offs_n[None, :] < N)
|
||||
tl.store(c_ptrs, c, mask=mask_c)
|
||||
|
||||
|
||||
# Per-shape (group_size, K, N) -> (BLOCK_M, BLOCK_N, BLOCK_K, num_warps,
|
||||
# num_stages) tile-config overrides for prefill (M <= 128) on gfx1x.
|
||||
# Picked by sweeping benchmarks/kernels/benchmark_rdna_hybrid_w4a16_gemm.py + a
|
||||
# per-config sweep script; only added when better than the generic heuristic
|
||||
# by > 20% at M=128. Re-run benchmarks after edits.
|
||||
_GFX1X_PREFILL_OVERRIDES: dict[tuple[int, int, int], tuple[int, int, int, int, int]] = {
|
||||
# SmolLM2-1.7B-Instruct-AWQ (gs=32, K=2048; gs forces BLOCK_K to 32 so
|
||||
# widen BLOCK_M and let Triton pipeline 4 stages to amortize the small
|
||||
# K-tile).
|
||||
(32, 2048, 6144): (128, 32, 32, 4, 4), # qkv_proj
|
||||
(32, 2048, 2048): (128, 32, 32, 4, 4), # o_proj
|
||||
(32, 2048, 16384): (128, 32, 32, 4, 4), # gate_up_proj
|
||||
(32, 8192, 2048): (128, 64, 32, 8, 2), # down_proj
|
||||
# Qwen3-8B-quantized.w4a16 (gs=128, K=4096 / 12288). For these K's a
|
||||
# full BLOCK_K=group_size with no software pipelining beats the generic
|
||||
# 64x64x64 — single-stage keeps register pressure down.
|
||||
(128, 4096, 6144): (128, 64, 128, 8, 1), # qkv_proj
|
||||
(128, 4096, 4096): (128, 32, 128, 4, 1), # o_proj
|
||||
(128, 4096, 24576): (64, 32, 128, 2, 1), # gate_up_proj
|
||||
(128, 12288, 4096): (128, 64, 128, 8, 1), # down_proj
|
||||
}
|
||||
|
||||
|
||||
def triton_w4a16_skinny_fmt_gemm(
|
||||
a: torch.Tensor, # [M, K] fp16/bf16
|
||||
b_q: torch.Tensor, # [N, K//8] int32 (ExLlama shuffle packed)
|
||||
scales: torch.Tensor, # [N, K//G] fp16/bf16
|
||||
group_size: int,
|
||||
zp_bias: int = 8,
|
||||
zp: torch.Tensor | None = None, # [N//8, K//G] int32 zero-points
|
||||
) -> torch.Tensor:
|
||||
"""Fused W4A16 GEMM reading from skinny weight format [N, K//8].
|
||||
|
||||
Args:
|
||||
a: Activation matrix [M, K], float16 or bfloat16.
|
||||
b_q: Packed weight matrix [N, K//8], int32 (ExLlama shuffle).
|
||||
scales: Per-group scales [N, K//G], same dtype as a.
|
||||
group_size: Quantization group size (resolved from -1 to K by caller).
|
||||
zp_bias: Constant zero bias (default 8 for unsigned int4).
|
||||
zp: Raw per-group zero-points [N//8, K//G] int32, row n at
|
||||
word[n//8] bits 4*(n%8) (asymmetric). When provided,
|
||||
dequant is (nibble - zp_raw) * scale.
|
||||
|
||||
Returns:
|
||||
Output matrix [M, N], same dtype as a.
|
||||
|
||||
"""
|
||||
assert a.stride(1) == 1, "Activation rows must be contiguous"
|
||||
assert b_q.stride(1) == 1, "Weight rows must be contiguous"
|
||||
assert scales.stride(1) == 1, "Scale rows must be contiguous"
|
||||
stride_sn = scales.stride(0)
|
||||
|
||||
M, K = a.shape
|
||||
N = b_q.shape[0]
|
||||
K8 = K // 8
|
||||
num_groups = K // group_size
|
||||
stride_bn = b_q.stride(0)
|
||||
stride_am = a.stride(0)
|
||||
|
||||
assert b_q.shape == (N, K8), f"b_q shape mismatch: {b_q.shape} vs ({N}, {K8})"
|
||||
assert scales.shape == (N, num_groups), (
|
||||
f"scales shape mismatch: {scales.shape} vs ({N}, {num_groups})"
|
||||
)
|
||||
if zp is not None:
|
||||
assert zp.stride(1) == 1, "Zero-point rows must be contiguous"
|
||||
assert zp.stride(0) == stride_sn, (
|
||||
f"zp row stride {zp.stride(0)} must match scales {stride_sn}"
|
||||
)
|
||||
assert N % 8 == 0, f"N must be divisible by 8 for packed zp, got {N}"
|
||||
assert zp.shape == (N // 8, num_groups), (
|
||||
f"zp shape mismatch: {zp.shape} vs ({N // 8}, {num_groups})"
|
||||
)
|
||||
has_zp = zp is not None
|
||||
|
||||
c = torch.empty((M, N), dtype=a.dtype, device=a.device)
|
||||
|
||||
# num_stages stays None unless a per-shape override sets it, so the
|
||||
# generic heuristics fall back to Triton's default pipeline depth.
|
||||
num_stages: int | None = None
|
||||
if _on_gfx12x():
|
||||
# Tuned on gfx1201 (Radeon AI PRO R9700, 32 CUs, 32-wide wavefronts)
|
||||
# using Llama-3.1-8B AWQ weight shapes with group_size=128.
|
||||
if M <= 32:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 16, 16, 128, 4
|
||||
elif M <= 64:
|
||||
if K >= 2 * N: # tall K (e.g. down_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 32, 128, 8
|
||||
elif N > K: # wide N (e.g. qkv_proj, gate_up_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 32, 64, 8
|
||||
else: # N ~= K (e.g. o_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 32, 64, 128, 4
|
||||
elif M <= 128:
|
||||
if K >= 2 * N: # tall K (e.g. down_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 16, 64, 1
|
||||
elif N >= 2 * K: # very wide N (e.g. gate_up_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 128, 64, 8
|
||||
else: # N ~= K (e.g. o_proj, qkv_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 64, 64, 8
|
||||
elif M <= 512:
|
||||
if K >= 2 * N: # tall K (e.g. down_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 64, 64, 8
|
||||
elif N >= 4 * K: # very wide N (e.g. gate_up_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 128, 64, 8
|
||||
else:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 128, 64, 8
|
||||
else:
|
||||
if K >= 2 * N: # tall K (e.g. down_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 64, 64, 8
|
||||
elif N >= 4 * K: # very wide N (e.g. gate_up_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 256, 64, 64, 8
|
||||
else:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 128, 32, 8
|
||||
elif _on_gfx1151():
|
||||
# Tuned on gfx1151 (Strix Halo, 40 CUs, 32-wide wavefronts)
|
||||
# using Qwen3-4B weight shapes with group_size=128.
|
||||
# Per-shape overrides for known prefill regressions live in a small
|
||||
# lookup table — see _GFX1X_PREFILL_OVERRIDES below. Re-run
|
||||
# benchmarks/kernels/benchmark_rdna_hybrid_w4a16_gemm.py after edits.
|
||||
override = (
|
||||
_GFX1X_PREFILL_OVERRIDES.get((group_size, K, N)) if M <= 128 else None
|
||||
)
|
||||
if override is not None:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages = override
|
||||
elif M <= 32:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 32, 32, 128, 4
|
||||
elif M <= 64:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 64, 32, 4
|
||||
elif M <= 128:
|
||||
if K >= 2 * N: # tall K (e.g. down_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 16, 64, 1
|
||||
elif N > K: # wide N (e.g. qkv_proj, gate_up_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 64, 64, 4
|
||||
else: # N ~= K (e.g. o_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 32, 64, 4
|
||||
elif M <= 1024:
|
||||
if K >= 2 * N: # tall K (e.g. down_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 64, 64, 4
|
||||
elif N >= 4 * K: # very wide N (e.g. gate_up_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 64, 64, 8
|
||||
else:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 128, 32, 4
|
||||
else:
|
||||
if K >= 2 * N: # tall K (e.g. down_proj)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 512, 32, 16
|
||||
else:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 64, 64, 8
|
||||
else:
|
||||
num_warps = 4
|
||||
if M <= 32:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 32, 64, 32
|
||||
elif M <= 64:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 64, 64, 32
|
||||
else:
|
||||
BLOCK_M, BLOCK_N, BLOCK_K = 128, 128, 32
|
||||
|
||||
# The kernel loads one scale per BLOCK_K tile, so BLOCK_K must not
|
||||
# exceed group_size — otherwise elements in the tile that belong to
|
||||
# a different group would get the wrong scale.
|
||||
BLOCK_K = min(BLOCK_K, group_size)
|
||||
|
||||
grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N))
|
||||
|
||||
extra_kwargs = {} if num_stages is None else {"num_stages": num_stages}
|
||||
_triton_w4a16_skinny_fmt_kernel[grid](
|
||||
a,
|
||||
b_q,
|
||||
scales,
|
||||
zp if has_zp else scales, # dummy pointer when no zp (unused)
|
||||
c,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
K8,
|
||||
num_groups,
|
||||
stride_sn,
|
||||
stride_bn,
|
||||
stride_am,
|
||||
group_size=group_size,
|
||||
ZP_BIAS=zp_bias,
|
||||
HAS_ZP=has_zp,
|
||||
BLOCK_M=BLOCK_M,
|
||||
BLOCK_N=BLOCK_N,
|
||||
BLOCK_K=BLOCK_K,
|
||||
num_warps=num_warps,
|
||||
**extra_kwargs,
|
||||
)
|
||||
return c
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Weight packing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def pack_int4_exllama_shuffle(w_uint4: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack uint4 values into ExLlama shuffle format: [N, K] -> [N, K//8] int32.
|
||||
|
||||
Each int32 packs 8 K-values with interleave order [0,2,4,6,1,3,5,7].
|
||||
"""
|
||||
N_dim, K_dim = w_uint4.shape
|
||||
assert K_dim % 8 == 0
|
||||
g = w_uint4.to(torch.uint8).view(N_dim, K_dim // 8, 8).to(torch.int32)
|
||||
return (
|
||||
g[:, :, 0]
|
||||
| (g[:, :, 2] << 4)
|
||||
| (g[:, :, 4] << 8)
|
||||
| (g[:, :, 6] << 12)
|
||||
| (g[:, :, 1] << 16)
|
||||
| (g[:, :, 3] << 20)
|
||||
| (g[:, :, 5] << 24)
|
||||
| (g[:, :, 7] << 28)
|
||||
)
|
||||
|
||||
|
||||
# gfx11 row strides alias in the vector caches when they land on a power-of-two
|
||||
# multiple.
|
||||
_WEIGHT_CLIFF_BYTES = 1024 # packed weight row, K/2 B
|
||||
_ACT_CLIFF_BYTES = 2048 # activation row, K*2 B
|
||||
_STRIDE_PAD_BYTES = 128 # one cache line
|
||||
|
||||
|
||||
def _weight_pad_bytes(row_bytes: int) -> int:
|
||||
"""Bytes to add to a packed weight row stride to move it off the cliff.
|
||||
|
||||
Only strides that are a multiple of ``_WEIGHT_CLIFF_BYTES`` are moved, and
|
||||
only by one cache line; padding a stride that is already off the cliff
|
||||
costs memory and measures slower.
|
||||
"""
|
||||
if row_bytes % _WEIGHT_CLIFF_BYTES:
|
||||
return 0
|
||||
return _STRIDE_PAD_BYTES
|
||||
|
||||
|
||||
def _act_pad_bytes(row_bytes: int) -> int:
|
||||
"""Bytes to add to an activation row stride to move it off the cliff."""
|
||||
if row_bytes % _ACT_CLIFF_BYTES:
|
||||
return 0
|
||||
return _STRIDE_PAD_BYTES
|
||||
|
||||
|
||||
def pack_skinny_int4(unpacked: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack [N, K] uint4 into the skinny weight layout the kernels consume.
|
||||
|
||||
ExLlama shuffle to [N, K//8] int32, viewed as int8 [N, K//2]. On gfx1151
|
||||
the row stride is nudged off the cliff (see ``_weight_pad_bytes``).
|
||||
"""
|
||||
shuffled = pack_int4_exllama_shuffle(unpacked)
|
||||
n_rows, k8 = shuffled.shape
|
||||
pad_int32 = _weight_pad_bytes(k8 * 4) // 4
|
||||
if not (pad_int32 and _on_gfx1151()):
|
||||
return shuffled.contiguous().view(torch.int8)
|
||||
padded = torch.empty(
|
||||
(n_rows, k8 + pad_int32), dtype=torch.int32, device=shuffled.device
|
||||
)
|
||||
padded[:, :k8].copy_(shuffled)
|
||||
# The int8 view keeps stride(0) = 4 * (k8 + pad_int32) bytes, and
|
||||
# .view(torch.int32) at apply time recovers the [N, K//8] int32 view.
|
||||
return padded.view(torch.int8)[:, : k8 * 4]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hybrid dispatch logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _pad_activation_rows(x_2d: torch.Tensor) -> torch.Tensor:
|
||||
"""Copy ``x_2d`` into a row-padded buffer when its row stride is on the cliff.
|
||||
|
||||
The packed weight is padded once at load time, but activations are produced
|
||||
fresh every step, so this materialises a padded copy.
|
||||
"""
|
||||
pad_bytes = _act_pad_bytes(x_2d.shape[1] * x_2d.element_size())
|
||||
if not (pad_bytes and _on_gfx1151()):
|
||||
return x_2d
|
||||
pad_elems = pad_bytes // x_2d.element_size()
|
||||
buf = torch.empty(
|
||||
(x_2d.shape[0], x_2d.shape[1] + pad_elems),
|
||||
dtype=x_2d.dtype,
|
||||
device=x_2d.device,
|
||||
)
|
||||
padded = buf[:, : x_2d.shape[1]]
|
||||
padded.copy_(x_2d)
|
||||
return padded
|
||||
|
||||
|
||||
def _rdna_hybrid_w4a16_apply_impl(
|
||||
x_2d: torch.Tensor,
|
||||
w_q: torch.Tensor,
|
||||
w_s: torch.Tensor,
|
||||
w_zp: torch.Tensor | None,
|
||||
bias: torch.Tensor | None,
|
||||
cu_count: int,
|
||||
group_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Dispatch between skinny GEMM and Triton based on batch size M.
|
||||
|
||||
``w_zp`` is [N//8, K//G] int32 for asymmetric layers, None for symmetric.
|
||||
"""
|
||||
import vllm._custom_ops as ops
|
||||
|
||||
M = x_2d.shape[0]
|
||||
K = x_2d.shape[1]
|
||||
N = w_q.shape[0]
|
||||
|
||||
if M <= MAX_SKINNY_BATCH_SIZE and (
|
||||
K * M <= MEDIUM_SKINNY_LIMIT_ELEMENTS or _on_gfx115x()
|
||||
):
|
||||
# record_function is not torch.compile-safe; use nullcontext when
|
||||
# compiling to keep the op traceable.
|
||||
ctx = (
|
||||
nullcontext()
|
||||
if torch.compiler.is_compiling()
|
||||
else torch.profiler.record_function(f"wvsplitk_int4 {M}x{N}x{K}")
|
||||
)
|
||||
with ctx:
|
||||
return ops.wvSplitK_int4_g(w_q, x_2d, w_s, cu_count, group_size, w_zp, bias)
|
||||
|
||||
ctx = (
|
||||
nullcontext()
|
||||
if torch.compiler.is_compiling()
|
||||
else torch.profiler.record_function(f"hybrid_triton_w4a16 {M}x{N}x{K}")
|
||||
)
|
||||
with ctx:
|
||||
output = triton_w4a16_skinny_fmt_gemm(
|
||||
a=_pad_activation_rows(x_2d),
|
||||
b_q=w_q.view(torch.int32),
|
||||
scales=w_s,
|
||||
group_size=group_size,
|
||||
zp=w_zp,
|
||||
)
|
||||
if bias is not None:
|
||||
output.add_(bias)
|
||||
return output
|
||||
|
||||
|
||||
def _rdna_hybrid_w4a16_apply_fake(
|
||||
x_2d: torch.Tensor,
|
||||
w_q: torch.Tensor,
|
||||
w_s: torch.Tensor,
|
||||
w_zp: torch.Tensor | None,
|
||||
bias: torch.Tensor | None,
|
||||
cu_count: int,
|
||||
group_size: int,
|
||||
) -> torch.Tensor:
|
||||
M = x_2d.size(0)
|
||||
N = w_q.size(0)
|
||||
return torch.empty((M, N), dtype=x_2d.dtype, device=x_2d.device)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rdna_hybrid_w4a16_apply",
|
||||
op_func=_rdna_hybrid_w4a16_apply_impl,
|
||||
mutates_args=[],
|
||||
fake_impl=_rdna_hybrid_w4a16_apply_fake,
|
||||
)
|
||||
|
||||
|
||||
class RDNAHybridW4A16LinearKernel(MPLinearKernel):
|
||||
"""Hybrid W4A16 kernel: HIP skinny for decode, Triton for prefill.
|
||||
|
||||
Stores the weights once as int8 [N, K//2] (ExLlama shuffle packed). The
|
||||
HIP skinny kernel reads it directly; the triton kernel reinterprets the
|
||||
same buffer as int32 [N, K//8] via a view, so there is no dual weight
|
||||
storage.
|
||||
"""
|
||||
|
||||
SUPPORTED_QUANT_TYPES = [
|
||||
scalar_types.uint4b8, # symmetric GPTQ (bias=8)
|
||||
scalar_types.uint4, # asymmetric (zero_points)
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
# Arch filtering is handled by can_implement (_on_gfx1x check)
|
||||
return 0
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_rocm():
|
||||
return False, "RDNAHybridW4A16LinearKernel only targets ROCm"
|
||||
|
||||
if not _on_gfx1x():
|
||||
return False, "RDNAHybridW4A16LinearKernel only targets gfx11/gfx12"
|
||||
|
||||
if c.weight_type not in cls.SUPPORTED_QUANT_TYPES:
|
||||
return (
|
||||
False,
|
||||
f"Quant type {c.weight_type} not supported; "
|
||||
f"supported: {cls.SUPPORTED_QUANT_TYPES}",
|
||||
)
|
||||
|
||||
if c.act_type not in (torch.float16, torch.bfloat16):
|
||||
return False, "requires float16 or bfloat16 activations"
|
||||
|
||||
gs = c.group_size
|
||||
if gs not in SUPPORTED_GROUP_SIZES:
|
||||
return (
|
||||
False,
|
||||
f"Group size {gs} not supported; supported: {SUPPORTED_GROUP_SIZES}",
|
||||
)
|
||||
|
||||
K = c.partition_weight_shape[0]
|
||||
if K % 16 != 0:
|
||||
return False, f"K={K} must be divisible by 16"
|
||||
|
||||
if K % gs != 0:
|
||||
return (
|
||||
False,
|
||||
f"K={K} must be divisible by group_size={gs}",
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
c = self.config
|
||||
|
||||
w_q_raw = getattr(layer, self.w_q_name)
|
||||
w_s_raw = getattr(layer, self.w_s_name)
|
||||
|
||||
unpacked = unpack_quantized_values_into_int32(
|
||||
w_q_raw.data, c.weight_type, packed_dim=w_q_raw.packed_dim
|
||||
)
|
||||
# AWQ weights arrive as (K, N) with output_dim=1;
|
||||
# compressed-tensors arrive as (N, K) with output_dim=0.
|
||||
if getattr(w_q_raw, "output_dim", 0) != 0:
|
||||
unpacked = unpacked.t().contiguous()
|
||||
|
||||
# Store as int8; Triton reinterprets via .view(torch.int32) at apply time.
|
||||
w_q_skinny = pack_skinny_int4(unpacked)
|
||||
|
||||
permute_param_layout_(w_s_raw, input_dim=1, output_dim=0)
|
||||
w_s_skinny = w_s_raw.data.contiguous()
|
||||
|
||||
if c.zero_points:
|
||||
assert self.w_zp_name is not None
|
||||
w_zp_raw = getattr(layer, self.w_zp_name)
|
||||
permute_param_layout_(w_zp_raw, input_dim=1, output_dim=0, packed_dim=0)
|
||||
w_zp = w_zp_raw.data.contiguous()
|
||||
self._transform_param(layer, self.w_zp_name, lambda x: w_zp)
|
||||
|
||||
self._transform_param(layer, self.w_q_name, lambda x: w_q_skinny)
|
||||
self._transform_param(layer, self.w_s_name, lambda x: w_s_skinny)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
c = self.config
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
|
||||
x_2d = x.reshape(-1, x.shape[-1])
|
||||
N = w_q.shape[0]
|
||||
out_shape = x.shape[:-1] + (N,)
|
||||
|
||||
cu_count = num_compute_units()
|
||||
output = torch.ops.vllm.rdna_hybrid_w4a16_apply(
|
||||
x_2d,
|
||||
w_q,
|
||||
w_s,
|
||||
w_zp,
|
||||
bias,
|
||||
cu_count,
|
||||
c.group_size,
|
||||
)
|
||||
return output.reshape(out_shape)
|
||||
@@ -1,737 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Portions adapted from ROCm/aiter (https://github.com/ROCm/aiter/pull/5750):
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
"""MXFP8 GEMM on 32x32 block-scaled weights for gfx950 (``tl.dot_scaled``).
|
||||
|
||||
``y = x @ w.T`` with an MXFP8 activation (e4m3 values, one E8M0 scale per
|
||||
row and 32 K, ``[M, K / 32]``) and an e4m3 weight whose E8M0 scales stay in
|
||||
the checkpoint's 32x32 blocks, ``[N / 32, K / 32]``, instead of being
|
||||
expanded to every row. Each weight scale byte is then read once per 32 output
|
||||
rows, and small-M shapes can use the packed kernel below.
|
||||
|
||||
Two kernels, picked per shape from a table tuned on MI355X:
|
||||
|
||||
* a tiled kernel for larger M;
|
||||
* a packed kernel for small M, which fills the MFMA's rows with K panels
|
||||
instead of tokens and keeps only the matching-panel (block-diagonal)
|
||||
products, so a handful of tokens still streams the weight at full width.
|
||||
|
||||
Either can split K. The partials are normally summed in the same launch by
|
||||
the last program of each output tile to finish, in split order, so the result
|
||||
does not depend on scheduling. Otherwise a second launch reduces them.
|
||||
|
||||
The packed kernel and the in-launch split-K reduction on one XCD
|
||||
(``_split_tile``, ``_sum_splits``, ``_split_counters``, ``_k_partition``) are
|
||||
adapted from the group32 GEMM in ROCm/aiter#5750.
|
||||
"""
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
BLOCK_ROWS = 32
|
||||
|
||||
# gfx950 hands out workgroups to its XCDs round-robin by program id, and 8 is
|
||||
# a multiple of the XCD count in every partition mode. _split_tile hardcodes
|
||||
# the same stride.
|
||||
_XCD_STRIDE = 8
|
||||
# One arrival counter per output tile of an in-launch split-K GEMM.
|
||||
_SPLIT_COUNTERS = 1 << 16
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _split_tile(N: tl.constexpr, BLOCK_N: tl.constexpr, SPLITS: tl.constexpr):
|
||||
"""(pid_m, pid_n, tile, split) of a 1D grid that keeps a tile's splits on
|
||||
one XCD, so they meet in that XCD's L2."""
|
||||
pid = tl.program_id(0)
|
||||
tile = pid // (8 * SPLITS) * 8 + pid % 8
|
||||
grid_n: tl.constexpr = (N + BLOCK_N - 1) // BLOCK_N
|
||||
return tile // grid_n, tile % grid_n, tile, pid // 8 % SPLITS
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sum_splits(
|
||||
acc, out_ptrs, out_mask, slot_ptr, count_ptr, split, SPLITS: tl.constexpr
|
||||
):
|
||||
"""Publish this split's partial; the tile's last arrival sums all of them
|
||||
in split order and re-arms the counter."""
|
||||
BM: tl.constexpr = acc.shape[0]
|
||||
BN: tl.constexpr = acc.shape[1]
|
||||
local = tl.arange(0, BM)[:, None] * BN + tl.arange(0, BN)[None, :]
|
||||
tl.store(slot_ptr + split * BM * BN + local, acc)
|
||||
# The partial must be in L2 before the arrival is counted. The splits
|
||||
# share an XCD, so no device-scope release (an L2 writeback) is needed.
|
||||
tl.inline_asm_elementwise(
|
||||
"s_waitcnt vmcnt(0)", "=v,v", [split], dtype=tl.int32, is_pure=False, pack=1
|
||||
)
|
||||
tl.debug_barrier()
|
||||
if tl.atomic_add(count_ptr, 1, sem="acq_rel", scope="cta") == SPLITS - 1:
|
||||
total = tl.zeros((BM, BN), dtype=tl.float32)
|
||||
for s in tl.static_range(SPLITS):
|
||||
total += tl.load(slot_ptr + s * BM * BN + local, cache_modifier=".cv")
|
||||
tl.store(out_ptrs, total.to(out_ptrs.dtype.element_ty), mask=out_mask)
|
||||
tl.store(count_ptr, 0)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["M"])
|
||||
def _block32_tiled_kernel(
|
||||
x_ptr,
|
||||
xs_ptr,
|
||||
w_ptr,
|
||||
ws_ptr,
|
||||
out_ptr,
|
||||
slot_ptr,
|
||||
count_ptr,
|
||||
M,
|
||||
N: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
stride_om,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
K_PER_SPLIT: tl.constexpr,
|
||||
N_FIRST: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
FUSED_SPLITS: tl.constexpr,
|
||||
):
|
||||
if FUSED_SPLITS > 1:
|
||||
pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS)
|
||||
if pid_m * BLOCK_M >= M:
|
||||
return
|
||||
else:
|
||||
# N_FIRST launches output-column tiles fastest, so consecutive
|
||||
# programs share an activation tile.
|
||||
if N_FIRST:
|
||||
pid_n, pid_m = tl.program_id(0), tl.program_id(1)
|
||||
else:
|
||||
pid_m, pid_n = tl.program_id(0), tl.program_id(1)
|
||||
pid_k = tl.program_id(2)
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
m_mask = offs_m < M
|
||||
n_mask = offs_n < N
|
||||
k0 = pid_k * K_PER_SPLIT
|
||||
offs_k = k0 + tl.arange(0, BLOCK_K)
|
||||
offs_sk = k0 // 32 + tl.arange(0, BLOCK_K // 32)
|
||||
x_ptrs = x_ptr + offs_m[:, None] * K + offs_k[None, :]
|
||||
xs_ptrs = xs_ptr + offs_m[:, None] * (K // 32) + offs_sk[None, :]
|
||||
w_ptrs = w_ptr + offs_n[:, None] * K + offs_k[None, :]
|
||||
ws_ptrs = ws_ptr + (offs_n[:, None] // 32) * (K // 32) + offs_sk[None, :]
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for kk in range(0, K_PER_SPLIT, BLOCK_K):
|
||||
if EVEN_K:
|
||||
x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0)
|
||||
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0)
|
||||
w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
|
||||
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0)
|
||||
else:
|
||||
k_ok = (offs_k + kk) < K
|
||||
s_ok = (offs_sk + kk // 32) < K // 32
|
||||
x = tl.load(x_ptrs, mask=m_mask[:, None] & k_ok[None, :], other=0.0)
|
||||
xs = tl.load(xs_ptrs, mask=m_mask[:, None] & s_ok[None, :], other=0)
|
||||
w = tl.load(w_ptrs, mask=n_mask[:, None] & k_ok[None, :], other=0.0)
|
||||
ws = tl.load(ws_ptrs, mask=n_mask[:, None] & s_ok[None, :], other=0)
|
||||
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
|
||||
x_ptrs += BLOCK_K
|
||||
w_ptrs += BLOCK_K
|
||||
xs_ptrs += BLOCK_K // 32
|
||||
ws_ptrs += BLOCK_K // 32
|
||||
out_mask = m_mask[:, None] & n_mask[None, :]
|
||||
if FUSED_SPLITS > 1:
|
||||
_sum_splits(
|
||||
acc,
|
||||
out_ptr + offs_m[:, None] * stride_om + offs_n[None, :],
|
||||
out_mask,
|
||||
slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N),
|
||||
count_ptr + tile,
|
||||
pid_k,
|
||||
FUSED_SPLITS,
|
||||
)
|
||||
else:
|
||||
out_ptrs = (
|
||||
out_ptr + pid_k * M * N + offs_m[:, None] * stride_om + offs_n[None, :]
|
||||
)
|
||||
tl.store(out_ptrs, acc.to(out_ptr.dtype.element_ty), mask=out_mask)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["M"])
|
||||
def _block32_packed_kernel(
|
||||
x_ptr,
|
||||
xs_ptr,
|
||||
w_ptr,
|
||||
ws_ptr,
|
||||
out_ptr,
|
||||
slot_ptr,
|
||||
count_ptr,
|
||||
M,
|
||||
N: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
stride_om,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
K_PACK: tl.constexpr,
|
||||
K_PER_SPLIT: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
FUSED_SPLITS: tl.constexpr,
|
||||
):
|
||||
if FUSED_SPLITS > 1:
|
||||
pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS)
|
||||
if pid_m * BLOCK_M >= M:
|
||||
return
|
||||
k_start = pid_k * K_PER_SPLIT
|
||||
k_stop = tl.minimum(k_start + K_PER_SPLIT, K)
|
||||
else:
|
||||
pid_m, pid_n, pid_k = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
if K_PER_SPLIT >= K:
|
||||
k_start = 0
|
||||
k_stop = K
|
||||
else:
|
||||
k_start = pid_k * K_PER_SPLIT
|
||||
k_stop = tl.minimum(k_start + K_PER_SPLIT, K)
|
||||
# Tile row r is token r // K_PACK reading K panel r % K_PACK (and likewise
|
||||
# for weight rows), so one dot covers K_PACK consecutive BLOCK_K slices.
|
||||
rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M * K_PACK) // K_PACK
|
||||
x_panel = tl.arange(0, BLOCK_M * K_PACK) % K_PACK
|
||||
cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N * K_PACK) // K_PACK
|
||||
w_panel = tl.arange(0, BLOCK_N * K_PACK) % K_PACK
|
||||
ks = tl.arange(0, BLOCK_K)
|
||||
gs = tl.arange(0, BLOCK_K // 32)
|
||||
row_ok = rows < M
|
||||
col_ok = cols < N
|
||||
acc = tl.zeros((BLOCK_M * K_PACK, BLOCK_N * K_PACK), dtype=tl.float32)
|
||||
for base in range(k_start, k_stop, BLOCK_K * K_PACK):
|
||||
xk = base + x_panel[:, None] * BLOCK_K + ks[None, :]
|
||||
wk = base + w_panel[:, None] * BLOCK_K + ks[None, :]
|
||||
xg = base // 32 + x_panel[:, None] * (BLOCK_K // 32) + gs[None, :]
|
||||
wg = base // 32 + w_panel[:, None] * (BLOCK_K // 32) + gs[None, :]
|
||||
x_mask = row_ok[:, None]
|
||||
w_mask = col_ok[:, None]
|
||||
xs_mask = row_ok[:, None]
|
||||
ws_mask = col_ok[:, None]
|
||||
if not EVEN_K:
|
||||
x_mask = x_mask & (xk < K)
|
||||
w_mask = w_mask & (wk < K)
|
||||
xs_mask = xs_mask & (xg < K // 32)
|
||||
ws_mask = ws_mask & (wg < K // 32)
|
||||
x = tl.load(x_ptr + rows[:, None] * K + xk, mask=x_mask, other=0.0)
|
||||
w = tl.load(w_ptr + cols[:, None] * K + wk, mask=w_mask, other=0.0)
|
||||
xs = tl.load(xs_ptr + rows[:, None] * (K // 32) + xg, mask=xs_mask, other=0)
|
||||
ws = tl.load(
|
||||
ws_ptr + (cols[:, None] // 32) * (K // 32) + wg, mask=ws_mask, other=0
|
||||
)
|
||||
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
|
||||
# Keep only products of matching K panels: the block diagonal.
|
||||
blocks = acc.reshape(BLOCK_M, K_PACK, BLOCK_N, K_PACK).trans(0, 2, 1, 3)
|
||||
pair = tl.arange(0, K_PACK)
|
||||
diagonal = tl.where(
|
||||
pair[None, None, :, None] == pair[None, None, None, :], blocks, 0.0
|
||||
)
|
||||
y = tl.sum(tl.sum(diagonal, 3), 2)
|
||||
out_rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
out_cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
out_ptrs = out_ptr + out_rows[:, None] * stride_om + out_cols[None, :]
|
||||
out_mask = (out_rows[:, None] < M) & (out_cols[None, :] < N)
|
||||
if FUSED_SPLITS > 1:
|
||||
_sum_splits(
|
||||
y,
|
||||
out_ptrs,
|
||||
out_mask,
|
||||
slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N),
|
||||
count_ptr + tile,
|
||||
pid_k,
|
||||
FUSED_SPLITS,
|
||||
)
|
||||
else:
|
||||
tl.store(
|
||||
out_ptrs + pid_k * M * N, y.to(out_ptr.dtype.element_ty), mask=out_mask
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["M"])
|
||||
def _block32_splitk_reduce_kernel(
|
||||
partial_ptr,
|
||||
out_ptr,
|
||||
M,
|
||||
N: tl.constexpr,
|
||||
stride_om,
|
||||
SPLIT_K: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
offs_m = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for split in tl.static_range(SPLIT_K):
|
||||
acc += tl.load(
|
||||
partial_ptr + split * M * N + offs_m[:, None] * N + offs_n[None, :],
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(
|
||||
out_ptr + offs_m[:, None] * stride_om + offs_n[None, :],
|
||||
acc.to(out_ptr.dtype.element_ty),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
class _Config(NamedTuple):
|
||||
packed: bool
|
||||
block_m: int
|
||||
block_n: int
|
||||
block_k: int
|
||||
# K panels per MFMA tile (packed) or K partitions reduced by a second
|
||||
# launch (tiled).
|
||||
k_split: int
|
||||
n_first: bool
|
||||
num_warps: int
|
||||
num_stages: int
|
||||
matrix_instr_nonkdim: int
|
||||
waves_per_eu: int
|
||||
# K partitions summed within the launch.
|
||||
fused_splits: int = 1
|
||||
|
||||
|
||||
def _tiled(
|
||||
bm, bn, bk, split_k, n_first, warps, stages, nonkdim, waves, fused=1
|
||||
) -> _Config:
|
||||
return _Config(
|
||||
False, bm, bn, bk, split_k, bool(n_first), warps, stages, nonkdim, waves, fused
|
||||
)
|
||||
|
||||
|
||||
def _packed(bm, bn, bk, k_pack, warps, stages, nonkdim, waves, fused=1) -> _Config:
|
||||
return _Config(
|
||||
True, bm, bn, bk, k_pack, False, warps, stages, nonkdim, waves, fused
|
||||
)
|
||||
|
||||
|
||||
# (N, K) -> [(max M, config), ...] for the DeepSeek-V4.1-Flash MXFP8 linears at
|
||||
# TP2/TP4, tuned on MI355X under HIP graphs with weights cold in the LLC. A
|
||||
# tier serves every M up to its bound; larger M reuse the last tier.
|
||||
_TUNED: dict[tuple[int, int], list[tuple[int, _Config]]] = {
|
||||
# shared expert gate_up, TP4
|
||||
(1152, 5120): [
|
||||
(1, _packed(8, 16, 1024, 2, 2, 2, 16, 1, 3)),
|
||||
(2, _packed(4, 32, 256, 4, 2, 2, 16, 1, 5)),
|
||||
(4, _packed(4, 32, 256, 4, 2, 2, 16, 0, 5)),
|
||||
(8, _packed(8, 16, 1024, 2, 2, 2, 16, 0, 3)),
|
||||
(16, _tiled(16, 32, 512, 1, True, 2, 3, 16, 0, 5)),
|
||||
(24, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1, 5)),
|
||||
(32, _packed(32, 32, 1024, 1, 4, 2, 16, 0, 5)),
|
||||
(48, _tiled(64, 32, 512, 1, True, 4, 3, 16, 1, 5)),
|
||||
(64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 5)),
|
||||
(96, _tiled(32, 16, 1024, 1, True, 2, 3, 16, 1)),
|
||||
(128, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 3)),
|
||||
(160, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)),
|
||||
(192, _tiled(32, 32, 1024, 1, True, 4, 2, 16, 0)),
|
||||
(256, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
|
||||
(320, _tiled(64, 32, 256, 1, True, 8, 3, 16, 0, 2)),
|
||||
(384, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)),
|
||||
(512, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
|
||||
(768, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
|
||||
(1024, _tiled(128, 128, 256, 2, False, 4, 2, 16, 1)),
|
||||
(1536, _tiled(128, 128, 256, 2, False, 4, 2, 16, 0)),
|
||||
(2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)),
|
||||
(3072, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
|
||||
],
|
||||
# fused wq_a|wkv (replicated)
|
||||
(1792, 5120): [
|
||||
(1, _packed(8, 16, 1024, 2, 2, 3, 16, 0)),
|
||||
(8, _packed(8, 16, 1024, 2, 2, 3, 16, 1)),
|
||||
(16, _tiled(16, 64, 1024, 1, True, 4, 2, 16, 0, 5)),
|
||||
(32, _packed(16, 16, 1024, 1, 2, 3, 16, 1)),
|
||||
(48, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 4)),
|
||||
(64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 4)),
|
||||
(96, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)),
|
||||
(128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 4)),
|
||||
(192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
|
||||
(256, _tiled(64, 64, 512, 1, True, 4, 2, 16, 1, 2)),
|
||||
(384, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
|
||||
(512, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
|
||||
(1024, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
|
||||
(3072, _tiled(128, 128, 256, 1, False, 8, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 128, 128, 1, False, 8, 2, 16, 1)),
|
||||
],
|
||||
# shared expert gate_up, TP2
|
||||
(2304, 5120): [
|
||||
(1, _packed(8, 16, 1024, 2, 2, 3, 16, 1)),
|
||||
(8, _packed(8, 16, 1024, 2, 2, 3, 16, 0)),
|
||||
(16, _tiled(16, 64, 1024, 1, True, 4, 3, 16, 1, 5)),
|
||||
(24, _packed(32, 32, 512, 1, 4, 2, 16, 0, 5)),
|
||||
(32, _packed(32, 32, 1024, 1, 4, 2, 16, 1, 3)),
|
||||
(48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 3)),
|
||||
(64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 5)),
|
||||
(128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
|
||||
(160, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 2)),
|
||||
(192, _tiled(64, 64, 512, 1, True, 4, 2, 16, 0, 2)),
|
||||
(256, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
|
||||
(320, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(512, _tiled(128, 128, 256, 1, True, 8, 2, 16, 0, 3)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
|
||||
(1536, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)),
|
||||
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)),
|
||||
(4096, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
|
||||
(8192, _tiled(128, 128, 128, 1, False, 8, 2, 32, 0)),
|
||||
],
|
||||
# indexer wq_b
|
||||
(4096, 1280): [
|
||||
(1, _packed(4, 16, 256, 4, 4, 2, 16, 1)),
|
||||
(2, _packed(4, 16, 256, 4, 4, 2, 16, 0)),
|
||||
(4, _packed(4, 16, 256, 4, 4, 2, 16, 1)),
|
||||
(8, _packed(8, 16, 256, 4, 4, 3, 16, 1)),
|
||||
(16, _packed(16, 16, 512, 2, 2, 3, 16, 1)),
|
||||
(32, _packed(32, 16, 1024, 1, 2, 3, 16, 1)),
|
||||
(64, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1)),
|
||||
(128, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0)),
|
||||
(192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0)),
|
||||
(256, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(384, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(512, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
|
||||
(1024, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)),
|
||||
],
|
||||
# shared expert down, TP4
|
||||
(5120, 576): [
|
||||
(8, _packed(8, 32, 256, 2, 2, 2, 16, 1)),
|
||||
(16, _packed(16, 32, 512, 1, 2, 2, 16, 0)),
|
||||
(32, _packed(32, 32, 256, 1, 8, 3, 16, 1)),
|
||||
(64, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)),
|
||||
(96, _tiled(32, 64, 256, 1, True, 4, 3, 16, 0)),
|
||||
(128, _tiled(32, 32, 256, 1, True, 2, 2, 16, 0)),
|
||||
(192, _tiled(64, 64, 256, 1, True, 4, 3, 32, 0)),
|
||||
(256, _tiled(64, 64, 256, 1, True, 2, 3, 32, 1)),
|
||||
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
|
||||
(512, _tiled(64, 64, 128, 1, True, 4, 2, 32, 0)),
|
||||
(768, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(1024, _tiled(128, 64, 128, 1, True, 4, 2, 32, 0)),
|
||||
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
|
||||
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
],
|
||||
# shared expert down, TP2
|
||||
(5120, 1152): [
|
||||
(1, _packed(4, 32, 128, 4, 2, 3, 16, 1)),
|
||||
(8, _packed(8, 32, 256, 2, 4, 3, 16, 0)),
|
||||
(32, _packed(32, 32, 512, 1, 2, 3, 16, 1)),
|
||||
(128, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)),
|
||||
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
|
||||
(512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
|
||||
(1024, _tiled(128, 64, 128, 1, True, 8, 2, 32, 0)),
|
||||
(1536, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(2048, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
|
||||
(3072, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
|
||||
(8192, _tiled(128, 64, 128, 1, True, 8, 2, 32, 1)),
|
||||
],
|
||||
# wo_b, TP4
|
||||
(5120, 2048): [
|
||||
(1, _packed(4, 32, 256, 4, 2, 2, 16, 0)),
|
||||
(4, _packed(4, 32, 256, 4, 2, 2, 16, 1)),
|
||||
(8, _packed(8, 32, 256, 4, 2, 3, 16, 0)),
|
||||
(16, _tiled(16, 32, 1024, 1, True, 2, 3, 16, 0)),
|
||||
(32, _tiled(32, 32, 512, 1, False, 4, 3, 16, 1)),
|
||||
(64, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)),
|
||||
(96, _tiled(32, 64, 512, 1, True, 8, 2, 16, 0)),
|
||||
(128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
|
||||
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
|
||||
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(8192, _tiled(128, 128, 128, 1, True, 2, 2, 32, 1)),
|
||||
],
|
||||
# wo_b, TP2
|
||||
(5120, 4096): [
|
||||
(4, _packed(4, 32, 512, 4, 2, 2, 16, 0)),
|
||||
(8, _packed(8, 16, 128, 4, 2, 2, 16, 0, 2)),
|
||||
(16, _tiled(16, 32, 256, 1, True, 2, 2, 16, 0, 4)),
|
||||
(24, _tiled(32, 32, 256, 1, True, 4, 3, 16, 0, 4)),
|
||||
(32, _tiled(32, 64, 512, 1, True, 8, 3, 16, 0, 3)),
|
||||
(48, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 3)),
|
||||
(64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 3)),
|
||||
(128, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
|
||||
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
|
||||
(256, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
|
||||
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(128, 128, 256, 1, True, 4, 2, 16, 0, 2)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)),
|
||||
(3072, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)),
|
||||
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
],
|
||||
# wq_b, TP4
|
||||
(8192, 1280): [
|
||||
(1, _packed(8, 32, 256, 2, 4, 3, 16, 1)),
|
||||
(8, _packed(8, 32, 256, 2, 4, 3, 16, 0)),
|
||||
(16, _packed(16, 32, 1024, 1, 2, 3, 16, 1)),
|
||||
(32, _packed(32, 32, 512, 1, 2, 3, 16, 0)),
|
||||
(48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1)),
|
||||
(64, _packed(32, 64, 256, 1, 4, 3, 16, 1)),
|
||||
(128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(192, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(256, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
|
||||
(384, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
|
||||
(1024, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(1536, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
|
||||
],
|
||||
# wq_b, TP2
|
||||
(16384, 1280): [
|
||||
(2, _packed(4, 32, 128, 4, 2, 3, 16, 1)),
|
||||
(4, _packed(4, 32, 128, 4, 4, 2, 16, 0)),
|
||||
(8, _packed(8, 32, 256, 2, 4, 2, 16, 0)),
|
||||
(32, _packed(32, 64, 256, 1, 4, 3, 16, 1)),
|
||||
(64, _tiled(64, 64, 256, 1, False, 4, 3, 32, 1)),
|
||||
(128, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
|
||||
(256, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(384, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(512, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
|
||||
(768, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)),
|
||||
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _default_config(M: int, N: int, K: int) -> _Config:
|
||||
"""Untuned shapes: the tiers most tuned shapes settle on."""
|
||||
if M <= 8:
|
||||
return _packed(8, 16, 256, 2, 2, 3, 16, 0)
|
||||
if M <= 32:
|
||||
return _packed(32, 32, 256, 1, 2, 3, 16, 0)
|
||||
if M <= 256:
|
||||
return _tiled(64, 64, 256, 1, True, 4, 2, 16, 0)
|
||||
return _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)
|
||||
|
||||
|
||||
def _config(M: int, N: int, K: int) -> _Config:
|
||||
tiers = _TUNED.get((N, K))
|
||||
if tiers is None:
|
||||
return _default_config(M, N, K)
|
||||
for max_m, cfg in tiers:
|
||||
if max_m >= M:
|
||||
return cfg
|
||||
return tiers[-1][1]
|
||||
|
||||
|
||||
_split_counters_by_stream: dict[tuple[int, int], torch.Tensor] = {}
|
||||
|
||||
|
||||
def _split_counters(device: torch.device) -> torch.Tensor | None:
|
||||
"""Zeroed arrival counters, one set per stream so concurrent launches never
|
||||
share a tile's counter. Each launch leaves the counters it used at zero.
|
||||
|
||||
None when a stream first needs them inside a CUDA graph capture: that
|
||||
allocation would come from the graph's pool, where it can take the address
|
||||
of an intermediate that every replay overwrites. vLLM warms up on its
|
||||
capture streams first, so this only affects captures without a warmup.
|
||||
"""
|
||||
key = (device.index, torch.accelerator.current_stream(device).stream_id)
|
||||
counters = _split_counters_by_stream.get(key)
|
||||
if counters is None:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
return None
|
||||
counters = torch.zeros(_SPLIT_COUNTERS, dtype=torch.int32, device=device)
|
||||
_split_counters_by_stream[key] = counters
|
||||
return counters
|
||||
|
||||
|
||||
def _k_partition(K: int, splits: int, step: int) -> tuple[int, int]:
|
||||
"""K per split, rounded up to whole steps, and the number of non-empty
|
||||
splits."""
|
||||
size = triton.cdiv(triton.cdiv(K, splits), step) * step
|
||||
return size, triton.cdiv(K, size)
|
||||
|
||||
|
||||
def _launch(
|
||||
x: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
cfg: _Config,
|
||||
) -> None:
|
||||
M, K = x.shape
|
||||
N = weight.shape[0]
|
||||
opts = dict(
|
||||
num_warps=cfg.num_warps,
|
||||
num_stages=cfg.num_stages,
|
||||
matrix_instr_nonkdim=cfg.matrix_instr_nonkdim,
|
||||
waves_per_eu=cfg.waves_per_eu,
|
||||
)
|
||||
grid_m, grid_n = triton.cdiv(M, cfg.block_m), triton.cdiv(N, cfg.block_n)
|
||||
tiles = grid_m * grid_n
|
||||
step = cfg.block_k * cfg.k_split if cfg.packed else cfg.block_k
|
||||
splits = cfg.fused_splits if cfg.fused_splits > 1 or cfg.packed else cfg.k_split
|
||||
k_per_split, splits = _k_partition(K, splits, step)
|
||||
counters = None
|
||||
if cfg.fused_splits > 1 and splits > 1 and tiles <= _SPLIT_COUNTERS:
|
||||
counters = _split_counters(out.device)
|
||||
if counters is not None:
|
||||
fused = splits
|
||||
target = out
|
||||
slots = torch.empty(
|
||||
tiles * fused * cfg.block_m * cfg.block_n,
|
||||
dtype=torch.float32,
|
||||
device=out.device,
|
||||
)
|
||||
grid: tuple[int, ...] = (triton.cdiv(tiles, _XCD_STRIDE) * _XCD_STRIDE * fused,)
|
||||
else:
|
||||
# Without counters, split K through a second launch.
|
||||
fused = 1
|
||||
target = (
|
||||
out
|
||||
if splits == 1
|
||||
else torch.empty((splits, M, N), dtype=torch.float32, device=x.device)
|
||||
)
|
||||
slots = counters = target # Unused by the kernels.
|
||||
if cfg.packed or not cfg.n_first:
|
||||
grid = (grid_m, grid_n, splits)
|
||||
else:
|
||||
grid = (grid_n, grid_m, splits)
|
||||
args = (x, x_scale, weight, weight_scale, target, slots, counters, M, N, K)
|
||||
stride = out.stride(0) if splits == 1 or fused > 1 else N
|
||||
if cfg.packed:
|
||||
_block32_packed_kernel[grid](
|
||||
*args,
|
||||
stride,
|
||||
BLOCK_M=cfg.block_m,
|
||||
BLOCK_N=cfg.block_n,
|
||||
BLOCK_K=cfg.block_k,
|
||||
K_PACK=cfg.k_split,
|
||||
K_PER_SPLIT=k_per_split,
|
||||
EVEN_K=K % step == 0,
|
||||
FUSED_SPLITS=fused,
|
||||
**opts,
|
||||
)
|
||||
else:
|
||||
_block32_tiled_kernel[grid](
|
||||
*args,
|
||||
stride,
|
||||
BLOCK_M=cfg.block_m,
|
||||
BLOCK_N=cfg.block_n,
|
||||
BLOCK_K=cfg.block_k,
|
||||
K_PER_SPLIT=k_per_split,
|
||||
N_FIRST=cfg.n_first,
|
||||
EVEN_K=K % k_per_split == 0,
|
||||
FUSED_SPLITS=fused,
|
||||
**opts,
|
||||
)
|
||||
if fused == 1 and splits > 1:
|
||||
_block32_splitk_reduce_kernel[(triton.cdiv(M, 32), triton.cdiv(N, 64))](
|
||||
target,
|
||||
out,
|
||||
M,
|
||||
N,
|
||||
out.stride(0),
|
||||
SPLIT_K=splits,
|
||||
BLOCK_M=32,
|
||||
BLOCK_N=64,
|
||||
)
|
||||
|
||||
|
||||
def _rocm_mxfp8_block32_gemm_impl(
|
||||
x: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
M, K = x.shape
|
||||
out = torch.empty((M, weight.shape[0]), dtype=out_dtype, device=x.device)
|
||||
if M > 0:
|
||||
_launch(x, x_scale, weight, weight_scale, out, _config(M, weight.shape[0], K))
|
||||
return out
|
||||
|
||||
|
||||
def _rocm_mxfp8_block32_gemm_fake(
|
||||
x: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
return x.new_empty((x.shape[0], weight.shape[0]), dtype=out_dtype)
|
||||
|
||||
|
||||
# An opaque op: the kernel and tile choice branch on the token count, which a
|
||||
# compiled caller must not freeze at its trace-time value.
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_mxfp8_block32_gemm",
|
||||
op_func=_rocm_mxfp8_block32_gemm_impl,
|
||||
fake_impl=_rocm_mxfp8_block32_gemm_fake,
|
||||
)
|
||||
|
||||
|
||||
def rocm_mxfp8_block32_gemm(
|
||||
x: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""``x @ weight.T`` for MXFP8 ``x`` and a 32x32 block-scaled MXFP8 weight.
|
||||
|
||||
Args:
|
||||
x: [M, K] e4m3 activation, contiguous.
|
||||
x_scale: [M, K / 32] E8M0 (uint8) activation scales.
|
||||
weight: [N, K] e4m3 weight, contiguous.
|
||||
weight_scale: [ceil(N / 32), K / 32] E8M0 (uint8) weight block scales.
|
||||
out_dtype: Output dtype.
|
||||
|
||||
Returns:
|
||||
The [M, N] product.
|
||||
|
||||
"""
|
||||
return torch.ops.vllm.rocm_mxfp8_block32_gemm(
|
||||
x, x_scale, weight, weight_scale, out_dtype
|
||||
)
|
||||
@@ -1,295 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Native MXFP8 linear GEMM for AMD CDNA4 (gfx950) via Triton ``tl.dot_scaled``.
|
||||
|
||||
Consumes the FP8 E4M3 weights + E8M0 block scales directly (no dequant-to-BF16);
|
||||
activations are MXFP8-quantized per token. Uses the CDNA4 hardware microscaling
|
||||
matrix cores. ``dot_scaled`` tiles K by 128; a weight whose K is not a multiple
|
||||
of that is dequantized to BF16 once at load and served by a plain linear
|
||||
instead, since ``can_implement`` does not filter on K.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
dequant_mxfp8_to_bf16,
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
from .rocm_block32_gemm import BLOCK_ROWS, rocm_mxfp8_block32_gemm
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mxfp8_linear_kernel(
|
||||
x_ptr,
|
||||
xs_ptr,
|
||||
w_ptr,
|
||||
ws_ptr,
|
||||
out_ptr,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
stride_xm,
|
||||
stride_xk,
|
||||
stride_xsm,
|
||||
stride_xsk,
|
||||
stride_wn,
|
||||
stride_wk,
|
||||
stride_wsn,
|
||||
stride_wsk,
|
||||
stride_om,
|
||||
stride_on,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0)
|
||||
pid_n = tl.program_id(1)
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
offs_k = tl.arange(0, BLOCK_K)
|
||||
offs_sk = tl.arange(0, BLOCK_K // 32)
|
||||
m_mask = offs_m < M
|
||||
n_mask = offs_n < N
|
||||
|
||||
x_ptrs = x_ptr + offs_m[:, None] * stride_xm + offs_k[None, :] * stride_xk
|
||||
xs_ptrs = xs_ptr + offs_m[:, None] * stride_xsm + offs_sk[None, :] * stride_xsk
|
||||
w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk
|
||||
ws_ptrs = ws_ptr + offs_n[:, None] * stride_wsn + offs_sk[None, :] * stride_wsk
|
||||
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for _ in range(0, tl.cdiv(K, BLOCK_K)):
|
||||
x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0)
|
||||
w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
|
||||
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0)
|
||||
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0)
|
||||
acc += tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3")
|
||||
x_ptrs += BLOCK_K * stride_xk
|
||||
w_ptrs += BLOCK_K * stride_wk
|
||||
xs_ptrs += (BLOCK_K // 32) * stride_xsk
|
||||
ws_ptrs += (BLOCK_K // 32) * stride_wsk
|
||||
|
||||
o_ptrs = out_ptr + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on
|
||||
tl.store(
|
||||
o_ptrs, acc.to(out_ptr.dtype.element_ty), mask=m_mask[:, None] & n_mask[None, :]
|
||||
)
|
||||
|
||||
|
||||
def _mxfp8_dot_scaled_linear(
|
||||
x: torch.Tensor, # [M, K] bf16/fp16
|
||||
w: torch.Tensor, # [N, K] fp8 e4m3
|
||||
w_scale: torch.Tensor, # [N, K//32] uint8 (E8M0)
|
||||
) -> torch.Tensor:
|
||||
M, K = x.shape
|
||||
N = w.shape[0]
|
||||
x_q, x_scale = mxfp8_e4m3_quantize(x)
|
||||
out = torch.empty((M, N), dtype=x.dtype, device=x.device)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages = _select_cfg(M, N, K)
|
||||
grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N))
|
||||
_mxfp8_linear_kernel[grid](
|
||||
x_q,
|
||||
x_scale,
|
||||
w,
|
||||
w_scale,
|
||||
out,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
x_q.stride(0),
|
||||
x_q.stride(1),
|
||||
x_scale.stride(0),
|
||||
x_scale.stride(1),
|
||||
w.stride(0),
|
||||
w.stride(1),
|
||||
w_scale.stride(0),
|
||||
w_scale.stride(1),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
BLOCK_M=BLOCK_M,
|
||||
BLOCK_N=BLOCK_N,
|
||||
BLOCK_K=BLOCK_K,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
# gfx950's async-copy LDS (default in the triton 3.8 this branch pins) leaves no
|
||||
# room for num_stages=3 at BLOCK_K=256; use 2 there.
|
||||
_BK256_STAGES = 3
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if on_gfx950():
|
||||
_BK256_STAGES = 2
|
||||
|
||||
|
||||
def _select_cfg(M, N, K):
|
||||
"""(BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages) — graph-tuned on gfx950.
|
||||
|
||||
The M-bucketed, shape-adaptive tile selection here is the speedup over the
|
||||
upstream 2-bucket launcher. Tiles are pipelined (num_stages>=2, larger BLOCK_K)
|
||||
and occupancy- and shape-aware: keyed on the LOCAL (M, N, K), so it adapts to the
|
||||
TP-sharded shapes (e.g. MiniMax-M3 TP=4 vs TP=8, where local N and K differ) —
|
||||
large-K prefill uses BLOCK_K=256; short-K (K=768) widens N. BLOCK_K must divide K
|
||||
(the K-loop is unmasked), so every BLOCK_K below is guarded to be K-divisible
|
||||
(served K: 384/768/1024/2048/6144).
|
||||
"""
|
||||
if M <= 64:
|
||||
# decode (M in {1,32,64}): tiny-M GEMV is weight-BW + GPU-OCCUPANCY bound. The
|
||||
# lever is NARROW BLOCK_N=16 (maximize N-tiles so more CUs stream the weight in
|
||||
# parallel) + LARGE BLOCK_K (fewer K-iters, bigger coalesced weight loads).
|
||||
# Tuned by CUDA-graph replay latency. Optimal at both TP=4 and TP=8.
|
||||
if K % 1024 == 0: # K=2048, 6144 -> graph-best 16x16x1024 (all M)
|
||||
return 16, 16, 1024, 2, 2
|
||||
if K % 512 == 0:
|
||||
return 16, 16, 512, 2, 3
|
||||
if K % 256 == 0: # K=768 (shared_down) -> graph-best 16x32x256
|
||||
return 16, 32, 256, 4, 3
|
||||
return 16, 32, 128, 4, 3
|
||||
# mid-M (65..256) on SMALL local-N: still occupancy-bound (a 64x64 tile makes too
|
||||
# few N-tiles), so the narrow-BLOCK_N decode-style tile fills the CUs better.
|
||||
# N<=1536 covers the real fused-qkv local N at TP=8: q heads shard but the GQA KV
|
||||
# (4) + sparse-indexer (4) heads are < TP=8, so vLLM replicates them to 1/rank ->
|
||||
# N = 1024 + 4*128 = 1536 (not 2560/2=1280). For the wider 1280<N<=1536 band the
|
||||
# narrow tile only wins up to M=128 (at M=256 the 64x64 tile is better), so cap it
|
||||
# there; N<=1280 keeps the narrow tile through M=256. TP=4 qkv N=2560 is unchanged.
|
||||
if (M <= 256 and N <= 1280) or (M <= 128 and N <= 1536):
|
||||
if K % 1024 == 0:
|
||||
return 16, 16, 1024, 2, 2
|
||||
if K % 512 == 0:
|
||||
return 16, 16, 512, 2, 3
|
||||
if K % 256 == 0:
|
||||
return 16, 16, 256, 2, 3
|
||||
return 16, 16, 128, 2, 3
|
||||
# right-sized launch grid (host-side), used to gate the tall 256-BLOCK_M tile.
|
||||
occ = triton.cdiv(M, 256) * triton.cdiv(N, 128)
|
||||
if K <= 1024: # short-K (shared_down K=384/768; TP=8 o_proj K=1024)
|
||||
if M <= 256:
|
||||
return (64, 64, 256, 8, 2) if K % 256 == 0 else (64, 64, 128, 8, 2)
|
||||
# large prefill. (The former 256x128x256 tall tile was faster only on triton
|
||||
# 3.6; on triton 3.7 its large BLOCK_M register/LDS footprint spills or hits
|
||||
# "out of resources", so use 128x128x256 -- within the known-good footprint.)
|
||||
if M >= 4096 and K >= 1024 and K % 256 == 0 and occ >= 256:
|
||||
return 128, 128, 256, 8, _BK256_STAGES
|
||||
return 128, 128, 128, 8, 3
|
||||
# large-K (K >= 2048). BLOCK_K is K-divisibility-guarded (the K-loop is unmasked):
|
||||
# served large-K is 2048/6144 (%256==0), but fall back to 128 (always divides, since
|
||||
# the entry requires K%128==0) for any other K to stay correct.
|
||||
if M <= 256: # conc~128 decode + small prefill chunk: occupancy tile
|
||||
if K % 512 == 0:
|
||||
return 64, 64, 512, 8, 2
|
||||
return (64, 64, 256, 8, 2) if K % 256 == 0 else (64, 64, 128, 8, 2)
|
||||
if M <= 1024: # medium prefill chunk: BN=64 keeps small-N occupied
|
||||
return (128, 64, 256, 8, 3) if K % 256 == 0 else (128, 64, 128, 8, 3)
|
||||
# large prefill (M > 1024). The previously graph-tuned tall 256x128x256 and deep
|
||||
# 128x128x512 tiles won only on triton 3.6; on triton 3.7 their larger BLOCK_M /
|
||||
# BLOCK_K register+LDS footprint spills (or hits "out of resources" on stricter
|
||||
# ROCm/triton builds). The 128x128x256 tile is equal-or-faster on triton 3.7 (the
|
||||
# M=4096,N=2048,K=6144 shape: ~104us vs the 256x128x256 tile's ~184us), within ~5%
|
||||
# on 3.6, and inside the footprint of the tiles used elsewhere in this selector.
|
||||
# Covers the qkv-class local N=1536 (TP=8 qkv / TP=4 shared_gate_up) and the deep-K
|
||||
# / very-large-M shapes.
|
||||
if K % 256 == 0 and (1280 < N <= 1536 or (occ >= 128 and (K >= 4096 or M >= 4096))):
|
||||
return 128, 128, 256, 8, _BK256_STAGES
|
||||
# small local-N (e.g. TP=8 shared_gate_up N=768): a 64-wide BLOCK_N doubles the
|
||||
# N-tile count -> better CU fill than 128x128 at this mid-large M (~1.4x there).
|
||||
if N <= 1024 and K % 256 == 0:
|
||||
return 128, 64, 256, 8, 3
|
||||
return (128, 128, 256, 8, 2) if K % 256 == 0 else (128, 256, 128, 8, 3)
|
||||
|
||||
|
||||
_DOT_SCALED_K_ALIGN = 128
|
||||
|
||||
|
||||
def _as_block32_scale(weight_scale: torch.Tensor) -> torch.Tensor | None:
|
||||
"""The [N / 32, K / 32] block scales, if every 32 rows share their scales."""
|
||||
N = weight_scale.shape[0]
|
||||
if N % BLOCK_ROWS != 0:
|
||||
return None
|
||||
blocks = weight_scale.view(N // BLOCK_ROWS, BLOCK_ROWS, -1)
|
||||
if not torch.equal(blocks, blocks[:, :1].expand_as(blocks)):
|
||||
return None
|
||||
return blocks[:, 0].contiguous()
|
||||
|
||||
|
||||
class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``."""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_rocm():
|
||||
return False, "not ROCm"
|
||||
# supports_mx() == gfx95x (CDNA4 native microscaling hardware). On other
|
||||
# archs dot_scaled would upcast to BF16, so the kernel selector falls
|
||||
# through to the BF16 emulation (hipBLASLt) path instead.
|
||||
if not current_platform.supports_mx():
|
||||
return False, "native MX requires CDNA4 (gfx95x)"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
weight = layer.weight.data # [N, K] fp8
|
||||
N, K = weight.shape
|
||||
scale_k = K // MXFP8_BLOCK_SIZE
|
||||
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
|
||||
# rocm_mxfp8_block32_gemm's split-K varies with M.
|
||||
block_scale = (
|
||||
None if envs.VLLM_BATCH_INVARIANT else _as_block32_scale(weight_scale)
|
||||
)
|
||||
if block_scale is not None:
|
||||
# Checkpoints with 32x32 scale blocks (DeepSeek V4/V4.1) get them
|
||||
# back from the loader expanded per row; keep the blocks and use
|
||||
# the block-scaled GEMM, which also handles any K % 32 == 0.
|
||||
weight_scale = block_scale
|
||||
elif K % _DOT_SCALED_K_ALIGN != 0:
|
||||
weight = dequant_mxfp8_to_bf16(weight.contiguous(), weight_scale)
|
||||
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
|
||||
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if layer.weight_scale.dtype != MXFP8_SCALE_DTYPE:
|
||||
raise ValueError(
|
||||
f"Expected {MXFP8_SCALE_DTYPE} weight_scale, got "
|
||||
f"{layer.weight_scale.dtype}."
|
||||
)
|
||||
out_shape = (*x.shape[:-1], layer.weight.shape[0])
|
||||
x2d = x.reshape(-1, x.shape[-1])
|
||||
if layer.weight_scale.shape[0] != layer.weight.shape[0]:
|
||||
# One scale row per 32 weight rows (see process_weights_after_loading).
|
||||
x_q, x_scale = mxfp8_e4m3_quantize(x2d)
|
||||
out = rocm_mxfp8_block32_gemm(
|
||||
x_q, x_scale, layer.weight, layer.weight_scale, x.dtype
|
||||
)
|
||||
elif layer.weight.element_size() >= 2:
|
||||
out = torch.nn.functional.linear(x2d, layer.weight.to(x.dtype))
|
||||
elif x2d.shape[-1] % _DOT_SCALED_K_ALIGN == 0:
|
||||
out = _mxfp8_dot_scaled_linear(x2d, layer.weight, layer.weight_scale)
|
||||
else:
|
||||
# process_weights_after_loading dequantizes these weights once, so
|
||||
# this runs only if it did not; dot_scaled would be invalid here.
|
||||
w_bf16 = dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale)
|
||||
out = torch.nn.functional.linear(x2d, w_bf16).to(x.dtype)
|
||||
out = out.reshape(out_shape)
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
return out
|
||||
@@ -28,9 +28,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.pytorch import (
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
RowWiseTorchFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.rocm import (
|
||||
ROCmFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearKernel,
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
@@ -62,7 +59,6 @@ __all__ = [
|
||||
"ChannelWiseTorchFP8ScaledMMLinearKernel",
|
||||
"PerTensorTorchFP8ScaledMMLinearKernel",
|
||||
"RowWiseTorchFP8ScaledMMLinearKernel",
|
||||
"ROCmFP8ScaledMMLinearKernel",
|
||||
"TritonFP8ScaledMMLinearKernel",
|
||||
"TritonInt8ScaledMMLinearKernel",
|
||||
"ZentorchInt8ScaledMMLinearKernel",
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
from .ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearKernel,
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
)
|
||||
|
||||
|
||||
def rocm_per_tensor_float_w8a8_scaled_mm_impl(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if (
|
||||
A.shape[0] <= 4
|
||||
and B.shape[0] % 16 == 0 # M TODO: needed?
|
||||
and B.shape[1] % 16 == 0 # K
|
||||
and ((bias is None) or (bias.dtype == out_dtype))
|
||||
):
|
||||
output = ops.wvSplitKQ(
|
||||
B.t(),
|
||||
A,
|
||||
out_dtype,
|
||||
As,
|
||||
Bs,
|
||||
num_compute_units(),
|
||||
bias,
|
||||
)
|
||||
# Fallback
|
||||
else:
|
||||
output = torch._scaled_mm(
|
||||
A,
|
||||
B,
|
||||
out_dtype=out_dtype,
|
||||
scale_a=As,
|
||||
scale_b=Bs,
|
||||
bias=bias,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def rocm_per_tensor_float_w8a8_scaled_mm_fake(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return A.new_empty((*A.shape[:-1], B.shape[1]), dtype=out_dtype)
|
||||
|
||||
|
||||
if current_platform.is_rocm():
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_per_tensor_float_w8a8_scaled_mm_impl",
|
||||
op_func=rocm_per_tensor_float_w8a8_scaled_mm_impl,
|
||||
fake_impl=rocm_per_tensor_float_w8a8_scaled_mm_fake,
|
||||
)
|
||||
|
||||
|
||||
class ROCmFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_rocm():
|
||||
return False, "requires ROCm."
|
||||
|
||||
from vllm.platforms.rocm import get_cdna_version, on_gfx12x, on_gfx1250
|
||||
|
||||
# wvSplitKQ (skinny GEMM) is excluded from the gfx1250 build.
|
||||
if on_gfx1250():
|
||||
return False, "wvSplitKQ (skinny GEMM) is not built on gfx1250"
|
||||
|
||||
# Restore RDNA4 (gfx12x) dropped by the get_cdna_version()>2 refactor.
|
||||
if get_cdna_version() <= 2 and not on_gfx12x():
|
||||
return False, "requires CDNA3+ (gfx942/gfx950) or RDNA4 (gfx12x)"
|
||||
|
||||
if not envs.VLLM_ROCM_USE_SKINNY_GEMM:
|
||||
return False, "requires VLLM_ROCM_USE_SKINNY_GEMM to be enabled."
|
||||
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
per_tensor_activation_scales = (
|
||||
c.activation_quant_key.scale.group_shape.is_per_tensor()
|
||||
)
|
||||
per_tensor_weight_scales = c.weight_quant_key.scale.group_shape.is_per_tensor()
|
||||
|
||||
if not (per_tensor_activation_scales and per_tensor_weight_scales):
|
||||
return False, "requires per tensor activation and weight scales."
|
||||
|
||||
return True, None
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
output = torch.ops.vllm.rocm_per_tensor_float_w8a8_scaled_mm_impl(
|
||||
A, B, out_dtype, As, Bs, bias
|
||||
)
|
||||
return torch.narrow(output, 0, 0, A.shape[0]).view(*output_shape)
|
||||
Reference in New Issue
Block a user