[Mini-VLLM] Remove the ROCm device path, debug routes and arch-gated tests
- Delete csrc/rocm, hipify, the _rocm_C extension and the setup.py rocm target path; drop the rocm requirement files - Delete the aiter, rdna3, skinny-gemm and ROCm fused-MoE kernel modules and retarget the linear kernel registries away from them - Reduce platforms/rocm.py to the GCN architecture predicates that live code still imports, and delete the RocmPlatform class and plugin - Drop the VLLM_SERVER_DEV_MODE debug surface: reset, collective_rpc, server_info and sleep routes plus the log_error_stack flag - Delete lm-format-enforcer, lark and otel-semantic-conventions-ai pins; the live server process maps none of them - Delete tests that only build architectures this fork no longer registers, and test directories left without any test file - Restore the tests/v1/kv_connector mock helpers that scheduler tests import at module level Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,737 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Portions adapted from ROCm/aiter (https://github.com/ROCm/aiter/pull/5750):
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
"""MXFP8 GEMM on 32x32 block-scaled weights for gfx950 (``tl.dot_scaled``).
|
||||
|
||||
``y = x @ w.T`` with an MXFP8 activation (e4m3 values, one E8M0 scale per
|
||||
row and 32 K, ``[M, K / 32]``) and an e4m3 weight whose E8M0 scales stay in
|
||||
the checkpoint's 32x32 blocks, ``[N / 32, K / 32]``, instead of being
|
||||
expanded to every row. Each weight scale byte is then read once per 32 output
|
||||
rows, and small-M shapes can use the packed kernel below.
|
||||
|
||||
Two kernels, picked per shape from a table tuned on MI355X:
|
||||
|
||||
* a tiled kernel for larger M;
|
||||
* a packed kernel for small M, which fills the MFMA's rows with K panels
|
||||
instead of tokens and keeps only the matching-panel (block-diagonal)
|
||||
products, so a handful of tokens still streams the weight at full width.
|
||||
|
||||
Either can split K. The partials are normally summed in the same launch by
|
||||
the last program of each output tile to finish, in split order, so the result
|
||||
does not depend on scheduling. Otherwise a second launch reduces them.
|
||||
|
||||
The packed kernel and the in-launch split-K reduction on one XCD
|
||||
(``_split_tile``, ``_sum_splits``, ``_split_counters``, ``_k_partition``) are
|
||||
adapted from the group32 GEMM in ROCm/aiter#5750.
|
||||
"""
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
BLOCK_ROWS = 32
|
||||
|
||||
# gfx950 hands out workgroups to its XCDs round-robin by program id, and 8 is
|
||||
# a multiple of the XCD count in every partition mode. _split_tile hardcodes
|
||||
# the same stride.
|
||||
_XCD_STRIDE = 8
|
||||
# One arrival counter per output tile of an in-launch split-K GEMM.
|
||||
_SPLIT_COUNTERS = 1 << 16
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _split_tile(N: tl.constexpr, BLOCK_N: tl.constexpr, SPLITS: tl.constexpr):
|
||||
"""(pid_m, pid_n, tile, split) of a 1D grid that keeps a tile's splits on
|
||||
one XCD, so they meet in that XCD's L2."""
|
||||
pid = tl.program_id(0)
|
||||
tile = pid // (8 * SPLITS) * 8 + pid % 8
|
||||
grid_n: tl.constexpr = (N + BLOCK_N - 1) // BLOCK_N
|
||||
return tile // grid_n, tile % grid_n, tile, pid // 8 % SPLITS
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sum_splits(
|
||||
acc, out_ptrs, out_mask, slot_ptr, count_ptr, split, SPLITS: tl.constexpr
|
||||
):
|
||||
"""Publish this split's partial; the tile's last arrival sums all of them
|
||||
in split order and re-arms the counter."""
|
||||
BM: tl.constexpr = acc.shape[0]
|
||||
BN: tl.constexpr = acc.shape[1]
|
||||
local = tl.arange(0, BM)[:, None] * BN + tl.arange(0, BN)[None, :]
|
||||
tl.store(slot_ptr + split * BM * BN + local, acc)
|
||||
# The partial must be in L2 before the arrival is counted. The splits
|
||||
# share an XCD, so no device-scope release (an L2 writeback) is needed.
|
||||
tl.inline_asm_elementwise(
|
||||
"s_waitcnt vmcnt(0)", "=v,v", [split], dtype=tl.int32, is_pure=False, pack=1
|
||||
)
|
||||
tl.debug_barrier()
|
||||
if tl.atomic_add(count_ptr, 1, sem="acq_rel", scope="cta") == SPLITS - 1:
|
||||
total = tl.zeros((BM, BN), dtype=tl.float32)
|
||||
for s in tl.static_range(SPLITS):
|
||||
total += tl.load(slot_ptr + s * BM * BN + local, cache_modifier=".cv")
|
||||
tl.store(out_ptrs, total.to(out_ptrs.dtype.element_ty), mask=out_mask)
|
||||
tl.store(count_ptr, 0)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["M"])
|
||||
def _block32_tiled_kernel(
|
||||
x_ptr,
|
||||
xs_ptr,
|
||||
w_ptr,
|
||||
ws_ptr,
|
||||
out_ptr,
|
||||
slot_ptr,
|
||||
count_ptr,
|
||||
M,
|
||||
N: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
stride_om,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
K_PER_SPLIT: tl.constexpr,
|
||||
N_FIRST: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
FUSED_SPLITS: tl.constexpr,
|
||||
):
|
||||
if FUSED_SPLITS > 1:
|
||||
pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS)
|
||||
if pid_m * BLOCK_M >= M:
|
||||
return
|
||||
else:
|
||||
# N_FIRST launches output-column tiles fastest, so consecutive
|
||||
# programs share an activation tile.
|
||||
if N_FIRST:
|
||||
pid_n, pid_m = tl.program_id(0), tl.program_id(1)
|
||||
else:
|
||||
pid_m, pid_n = tl.program_id(0), tl.program_id(1)
|
||||
pid_k = tl.program_id(2)
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
m_mask = offs_m < M
|
||||
n_mask = offs_n < N
|
||||
k0 = pid_k * K_PER_SPLIT
|
||||
offs_k = k0 + tl.arange(0, BLOCK_K)
|
||||
offs_sk = k0 // 32 + tl.arange(0, BLOCK_K // 32)
|
||||
x_ptrs = x_ptr + offs_m[:, None] * K + offs_k[None, :]
|
||||
xs_ptrs = xs_ptr + offs_m[:, None] * (K // 32) + offs_sk[None, :]
|
||||
w_ptrs = w_ptr + offs_n[:, None] * K + offs_k[None, :]
|
||||
ws_ptrs = ws_ptr + (offs_n[:, None] // 32) * (K // 32) + offs_sk[None, :]
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for kk in range(0, K_PER_SPLIT, BLOCK_K):
|
||||
if EVEN_K:
|
||||
x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0)
|
||||
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0)
|
||||
w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
|
||||
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0)
|
||||
else:
|
||||
k_ok = (offs_k + kk) < K
|
||||
s_ok = (offs_sk + kk // 32) < K // 32
|
||||
x = tl.load(x_ptrs, mask=m_mask[:, None] & k_ok[None, :], other=0.0)
|
||||
xs = tl.load(xs_ptrs, mask=m_mask[:, None] & s_ok[None, :], other=0)
|
||||
w = tl.load(w_ptrs, mask=n_mask[:, None] & k_ok[None, :], other=0.0)
|
||||
ws = tl.load(ws_ptrs, mask=n_mask[:, None] & s_ok[None, :], other=0)
|
||||
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
|
||||
x_ptrs += BLOCK_K
|
||||
w_ptrs += BLOCK_K
|
||||
xs_ptrs += BLOCK_K // 32
|
||||
ws_ptrs += BLOCK_K // 32
|
||||
out_mask = m_mask[:, None] & n_mask[None, :]
|
||||
if FUSED_SPLITS > 1:
|
||||
_sum_splits(
|
||||
acc,
|
||||
out_ptr + offs_m[:, None] * stride_om + offs_n[None, :],
|
||||
out_mask,
|
||||
slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N),
|
||||
count_ptr + tile,
|
||||
pid_k,
|
||||
FUSED_SPLITS,
|
||||
)
|
||||
else:
|
||||
out_ptrs = (
|
||||
out_ptr + pid_k * M * N + offs_m[:, None] * stride_om + offs_n[None, :]
|
||||
)
|
||||
tl.store(out_ptrs, acc.to(out_ptr.dtype.element_ty), mask=out_mask)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["M"])
|
||||
def _block32_packed_kernel(
|
||||
x_ptr,
|
||||
xs_ptr,
|
||||
w_ptr,
|
||||
ws_ptr,
|
||||
out_ptr,
|
||||
slot_ptr,
|
||||
count_ptr,
|
||||
M,
|
||||
N: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
stride_om,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
K_PACK: tl.constexpr,
|
||||
K_PER_SPLIT: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
FUSED_SPLITS: tl.constexpr,
|
||||
):
|
||||
if FUSED_SPLITS > 1:
|
||||
pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS)
|
||||
if pid_m * BLOCK_M >= M:
|
||||
return
|
||||
k_start = pid_k * K_PER_SPLIT
|
||||
k_stop = tl.minimum(k_start + K_PER_SPLIT, K)
|
||||
else:
|
||||
pid_m, pid_n, pid_k = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
if K_PER_SPLIT >= K:
|
||||
k_start = 0
|
||||
k_stop = K
|
||||
else:
|
||||
k_start = pid_k * K_PER_SPLIT
|
||||
k_stop = tl.minimum(k_start + K_PER_SPLIT, K)
|
||||
# Tile row r is token r // K_PACK reading K panel r % K_PACK (and likewise
|
||||
# for weight rows), so one dot covers K_PACK consecutive BLOCK_K slices.
|
||||
rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M * K_PACK) // K_PACK
|
||||
x_panel = tl.arange(0, BLOCK_M * K_PACK) % K_PACK
|
||||
cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N * K_PACK) // K_PACK
|
||||
w_panel = tl.arange(0, BLOCK_N * K_PACK) % K_PACK
|
||||
ks = tl.arange(0, BLOCK_K)
|
||||
gs = tl.arange(0, BLOCK_K // 32)
|
||||
row_ok = rows < M
|
||||
col_ok = cols < N
|
||||
acc = tl.zeros((BLOCK_M * K_PACK, BLOCK_N * K_PACK), dtype=tl.float32)
|
||||
for base in range(k_start, k_stop, BLOCK_K * K_PACK):
|
||||
xk = base + x_panel[:, None] * BLOCK_K + ks[None, :]
|
||||
wk = base + w_panel[:, None] * BLOCK_K + ks[None, :]
|
||||
xg = base // 32 + x_panel[:, None] * (BLOCK_K // 32) + gs[None, :]
|
||||
wg = base // 32 + w_panel[:, None] * (BLOCK_K // 32) + gs[None, :]
|
||||
x_mask = row_ok[:, None]
|
||||
w_mask = col_ok[:, None]
|
||||
xs_mask = row_ok[:, None]
|
||||
ws_mask = col_ok[:, None]
|
||||
if not EVEN_K:
|
||||
x_mask = x_mask & (xk < K)
|
||||
w_mask = w_mask & (wk < K)
|
||||
xs_mask = xs_mask & (xg < K // 32)
|
||||
ws_mask = ws_mask & (wg < K // 32)
|
||||
x = tl.load(x_ptr + rows[:, None] * K + xk, mask=x_mask, other=0.0)
|
||||
w = tl.load(w_ptr + cols[:, None] * K + wk, mask=w_mask, other=0.0)
|
||||
xs = tl.load(xs_ptr + rows[:, None] * (K // 32) + xg, mask=xs_mask, other=0)
|
||||
ws = tl.load(
|
||||
ws_ptr + (cols[:, None] // 32) * (K // 32) + wg, mask=ws_mask, other=0
|
||||
)
|
||||
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
|
||||
# Keep only products of matching K panels: the block diagonal.
|
||||
blocks = acc.reshape(BLOCK_M, K_PACK, BLOCK_N, K_PACK).trans(0, 2, 1, 3)
|
||||
pair = tl.arange(0, K_PACK)
|
||||
diagonal = tl.where(
|
||||
pair[None, None, :, None] == pair[None, None, None, :], blocks, 0.0
|
||||
)
|
||||
y = tl.sum(tl.sum(diagonal, 3), 2)
|
||||
out_rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
out_cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
out_ptrs = out_ptr + out_rows[:, None] * stride_om + out_cols[None, :]
|
||||
out_mask = (out_rows[:, None] < M) & (out_cols[None, :] < N)
|
||||
if FUSED_SPLITS > 1:
|
||||
_sum_splits(
|
||||
y,
|
||||
out_ptrs,
|
||||
out_mask,
|
||||
slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N),
|
||||
count_ptr + tile,
|
||||
pid_k,
|
||||
FUSED_SPLITS,
|
||||
)
|
||||
else:
|
||||
tl.store(
|
||||
out_ptrs + pid_k * M * N, y.to(out_ptr.dtype.element_ty), mask=out_mask
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["M"])
|
||||
def _block32_splitk_reduce_kernel(
|
||||
partial_ptr,
|
||||
out_ptr,
|
||||
M,
|
||||
N: tl.constexpr,
|
||||
stride_om,
|
||||
SPLIT_K: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
offs_m = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for split in tl.static_range(SPLIT_K):
|
||||
acc += tl.load(
|
||||
partial_ptr + split * M * N + offs_m[:, None] * N + offs_n[None, :],
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(
|
||||
out_ptr + offs_m[:, None] * stride_om + offs_n[None, :],
|
||||
acc.to(out_ptr.dtype.element_ty),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
class _Config(NamedTuple):
|
||||
packed: bool
|
||||
block_m: int
|
||||
block_n: int
|
||||
block_k: int
|
||||
# K panels per MFMA tile (packed) or K partitions reduced by a second
|
||||
# launch (tiled).
|
||||
k_split: int
|
||||
n_first: bool
|
||||
num_warps: int
|
||||
num_stages: int
|
||||
matrix_instr_nonkdim: int
|
||||
waves_per_eu: int
|
||||
# K partitions summed within the launch.
|
||||
fused_splits: int = 1
|
||||
|
||||
|
||||
def _tiled(
|
||||
bm, bn, bk, split_k, n_first, warps, stages, nonkdim, waves, fused=1
|
||||
) -> _Config:
|
||||
return _Config(
|
||||
False, bm, bn, bk, split_k, bool(n_first), warps, stages, nonkdim, waves, fused
|
||||
)
|
||||
|
||||
|
||||
def _packed(bm, bn, bk, k_pack, warps, stages, nonkdim, waves, fused=1) -> _Config:
|
||||
return _Config(
|
||||
True, bm, bn, bk, k_pack, False, warps, stages, nonkdim, waves, fused
|
||||
)
|
||||
|
||||
|
||||
# (N, K) -> [(max M, config), ...] for the DeepSeek-V4.1-Flash MXFP8 linears at
|
||||
# TP2/TP4, tuned on MI355X under HIP graphs with weights cold in the LLC. A
|
||||
# tier serves every M up to its bound; larger M reuse the last tier.
|
||||
_TUNED: dict[tuple[int, int], list[tuple[int, _Config]]] = {
|
||||
# shared expert gate_up, TP4
|
||||
(1152, 5120): [
|
||||
(1, _packed(8, 16, 1024, 2, 2, 2, 16, 1, 3)),
|
||||
(2, _packed(4, 32, 256, 4, 2, 2, 16, 1, 5)),
|
||||
(4, _packed(4, 32, 256, 4, 2, 2, 16, 0, 5)),
|
||||
(8, _packed(8, 16, 1024, 2, 2, 2, 16, 0, 3)),
|
||||
(16, _tiled(16, 32, 512, 1, True, 2, 3, 16, 0, 5)),
|
||||
(24, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1, 5)),
|
||||
(32, _packed(32, 32, 1024, 1, 4, 2, 16, 0, 5)),
|
||||
(48, _tiled(64, 32, 512, 1, True, 4, 3, 16, 1, 5)),
|
||||
(64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 5)),
|
||||
(96, _tiled(32, 16, 1024, 1, True, 2, 3, 16, 1)),
|
||||
(128, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 3)),
|
||||
(160, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)),
|
||||
(192, _tiled(32, 32, 1024, 1, True, 4, 2, 16, 0)),
|
||||
(256, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
|
||||
(320, _tiled(64, 32, 256, 1, True, 8, 3, 16, 0, 2)),
|
||||
(384, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)),
|
||||
(512, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
|
||||
(768, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
|
||||
(1024, _tiled(128, 128, 256, 2, False, 4, 2, 16, 1)),
|
||||
(1536, _tiled(128, 128, 256, 2, False, 4, 2, 16, 0)),
|
||||
(2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)),
|
||||
(3072, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
|
||||
],
|
||||
# fused wq_a|wkv (replicated)
|
||||
(1792, 5120): [
|
||||
(1, _packed(8, 16, 1024, 2, 2, 3, 16, 0)),
|
||||
(8, _packed(8, 16, 1024, 2, 2, 3, 16, 1)),
|
||||
(16, _tiled(16, 64, 1024, 1, True, 4, 2, 16, 0, 5)),
|
||||
(32, _packed(16, 16, 1024, 1, 2, 3, 16, 1)),
|
||||
(48, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 4)),
|
||||
(64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 4)),
|
||||
(96, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)),
|
||||
(128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 4)),
|
||||
(192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
|
||||
(256, _tiled(64, 64, 512, 1, True, 4, 2, 16, 1, 2)),
|
||||
(384, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
|
||||
(512, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
|
||||
(1024, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
|
||||
(3072, _tiled(128, 128, 256, 1, False, 8, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 128, 128, 1, False, 8, 2, 16, 1)),
|
||||
],
|
||||
# shared expert gate_up, TP2
|
||||
(2304, 5120): [
|
||||
(1, _packed(8, 16, 1024, 2, 2, 3, 16, 1)),
|
||||
(8, _packed(8, 16, 1024, 2, 2, 3, 16, 0)),
|
||||
(16, _tiled(16, 64, 1024, 1, True, 4, 3, 16, 1, 5)),
|
||||
(24, _packed(32, 32, 512, 1, 4, 2, 16, 0, 5)),
|
||||
(32, _packed(32, 32, 1024, 1, 4, 2, 16, 1, 3)),
|
||||
(48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 3)),
|
||||
(64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 5)),
|
||||
(128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
|
||||
(160, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 2)),
|
||||
(192, _tiled(64, 64, 512, 1, True, 4, 2, 16, 0, 2)),
|
||||
(256, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
|
||||
(320, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(512, _tiled(128, 128, 256, 1, True, 8, 2, 16, 0, 3)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
|
||||
(1536, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)),
|
||||
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)),
|
||||
(4096, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
|
||||
(8192, _tiled(128, 128, 128, 1, False, 8, 2, 32, 0)),
|
||||
],
|
||||
# indexer wq_b
|
||||
(4096, 1280): [
|
||||
(1, _packed(4, 16, 256, 4, 4, 2, 16, 1)),
|
||||
(2, _packed(4, 16, 256, 4, 4, 2, 16, 0)),
|
||||
(4, _packed(4, 16, 256, 4, 4, 2, 16, 1)),
|
||||
(8, _packed(8, 16, 256, 4, 4, 3, 16, 1)),
|
||||
(16, _packed(16, 16, 512, 2, 2, 3, 16, 1)),
|
||||
(32, _packed(32, 16, 1024, 1, 2, 3, 16, 1)),
|
||||
(64, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1)),
|
||||
(128, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0)),
|
||||
(192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0)),
|
||||
(256, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(384, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(512, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
|
||||
(1024, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)),
|
||||
],
|
||||
# shared expert down, TP4
|
||||
(5120, 576): [
|
||||
(8, _packed(8, 32, 256, 2, 2, 2, 16, 1)),
|
||||
(16, _packed(16, 32, 512, 1, 2, 2, 16, 0)),
|
||||
(32, _packed(32, 32, 256, 1, 8, 3, 16, 1)),
|
||||
(64, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)),
|
||||
(96, _tiled(32, 64, 256, 1, True, 4, 3, 16, 0)),
|
||||
(128, _tiled(32, 32, 256, 1, True, 2, 2, 16, 0)),
|
||||
(192, _tiled(64, 64, 256, 1, True, 4, 3, 32, 0)),
|
||||
(256, _tiled(64, 64, 256, 1, True, 2, 3, 32, 1)),
|
||||
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
|
||||
(512, _tiled(64, 64, 128, 1, True, 4, 2, 32, 0)),
|
||||
(768, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(1024, _tiled(128, 64, 128, 1, True, 4, 2, 32, 0)),
|
||||
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
|
||||
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
],
|
||||
# shared expert down, TP2
|
||||
(5120, 1152): [
|
||||
(1, _packed(4, 32, 128, 4, 2, 3, 16, 1)),
|
||||
(8, _packed(8, 32, 256, 2, 4, 3, 16, 0)),
|
||||
(32, _packed(32, 32, 512, 1, 2, 3, 16, 1)),
|
||||
(128, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)),
|
||||
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
|
||||
(512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
|
||||
(1024, _tiled(128, 64, 128, 1, True, 8, 2, 32, 0)),
|
||||
(1536, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(2048, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
|
||||
(3072, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
|
||||
(8192, _tiled(128, 64, 128, 1, True, 8, 2, 32, 1)),
|
||||
],
|
||||
# wo_b, TP4
|
||||
(5120, 2048): [
|
||||
(1, _packed(4, 32, 256, 4, 2, 2, 16, 0)),
|
||||
(4, _packed(4, 32, 256, 4, 2, 2, 16, 1)),
|
||||
(8, _packed(8, 32, 256, 4, 2, 3, 16, 0)),
|
||||
(16, _tiled(16, 32, 1024, 1, True, 2, 3, 16, 0)),
|
||||
(32, _tiled(32, 32, 512, 1, False, 4, 3, 16, 1)),
|
||||
(64, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)),
|
||||
(96, _tiled(32, 64, 512, 1, True, 8, 2, 16, 0)),
|
||||
(128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
|
||||
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
|
||||
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(8192, _tiled(128, 128, 128, 1, True, 2, 2, 32, 1)),
|
||||
],
|
||||
# wo_b, TP2
|
||||
(5120, 4096): [
|
||||
(4, _packed(4, 32, 512, 4, 2, 2, 16, 0)),
|
||||
(8, _packed(8, 16, 128, 4, 2, 2, 16, 0, 2)),
|
||||
(16, _tiled(16, 32, 256, 1, True, 2, 2, 16, 0, 4)),
|
||||
(24, _tiled(32, 32, 256, 1, True, 4, 3, 16, 0, 4)),
|
||||
(32, _tiled(32, 64, 512, 1, True, 8, 3, 16, 0, 3)),
|
||||
(48, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 3)),
|
||||
(64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 3)),
|
||||
(128, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
|
||||
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
|
||||
(256, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
|
||||
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(128, 128, 256, 1, True, 4, 2, 16, 0, 2)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)),
|
||||
(3072, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)),
|
||||
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
],
|
||||
# wq_b, TP4
|
||||
(8192, 1280): [
|
||||
(1, _packed(8, 32, 256, 2, 4, 3, 16, 1)),
|
||||
(8, _packed(8, 32, 256, 2, 4, 3, 16, 0)),
|
||||
(16, _packed(16, 32, 1024, 1, 2, 3, 16, 1)),
|
||||
(32, _packed(32, 32, 512, 1, 2, 3, 16, 0)),
|
||||
(48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1)),
|
||||
(64, _packed(32, 64, 256, 1, 4, 3, 16, 1)),
|
||||
(128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(192, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(256, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
|
||||
(384, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
|
||||
(1024, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(1536, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
|
||||
],
|
||||
# wq_b, TP2
|
||||
(16384, 1280): [
|
||||
(2, _packed(4, 32, 128, 4, 2, 3, 16, 1)),
|
||||
(4, _packed(4, 32, 128, 4, 4, 2, 16, 0)),
|
||||
(8, _packed(8, 32, 256, 2, 4, 2, 16, 0)),
|
||||
(32, _packed(32, 64, 256, 1, 4, 3, 16, 1)),
|
||||
(64, _tiled(64, 64, 256, 1, False, 4, 3, 32, 1)),
|
||||
(128, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
|
||||
(256, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(384, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(512, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
|
||||
(768, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)),
|
||||
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _default_config(M: int, N: int, K: int) -> _Config:
|
||||
"""Untuned shapes: the tiers most tuned shapes settle on."""
|
||||
if M <= 8:
|
||||
return _packed(8, 16, 256, 2, 2, 3, 16, 0)
|
||||
if M <= 32:
|
||||
return _packed(32, 32, 256, 1, 2, 3, 16, 0)
|
||||
if M <= 256:
|
||||
return _tiled(64, 64, 256, 1, True, 4, 2, 16, 0)
|
||||
return _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)
|
||||
|
||||
|
||||
def _config(M: int, N: int, K: int) -> _Config:
|
||||
tiers = _TUNED.get((N, K))
|
||||
if tiers is None:
|
||||
return _default_config(M, N, K)
|
||||
for max_m, cfg in tiers:
|
||||
if max_m >= M:
|
||||
return cfg
|
||||
return tiers[-1][1]
|
||||
|
||||
|
||||
_split_counters_by_stream: dict[tuple[int, int], torch.Tensor] = {}
|
||||
|
||||
|
||||
def _split_counters(device: torch.device) -> torch.Tensor | None:
|
||||
"""Zeroed arrival counters, one set per stream so concurrent launches never
|
||||
share a tile's counter. Each launch leaves the counters it used at zero.
|
||||
|
||||
None when a stream first needs them inside a CUDA graph capture: that
|
||||
allocation would come from the graph's pool, where it can take the address
|
||||
of an intermediate that every replay overwrites. vLLM warms up on its
|
||||
capture streams first, so this only affects captures without a warmup.
|
||||
"""
|
||||
key = (device.index, torch.accelerator.current_stream(device).stream_id)
|
||||
counters = _split_counters_by_stream.get(key)
|
||||
if counters is None:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
return None
|
||||
counters = torch.zeros(_SPLIT_COUNTERS, dtype=torch.int32, device=device)
|
||||
_split_counters_by_stream[key] = counters
|
||||
return counters
|
||||
|
||||
|
||||
def _k_partition(K: int, splits: int, step: int) -> tuple[int, int]:
|
||||
"""K per split, rounded up to whole steps, and the number of non-empty
|
||||
splits."""
|
||||
size = triton.cdiv(triton.cdiv(K, splits), step) * step
|
||||
return size, triton.cdiv(K, size)
|
||||
|
||||
|
||||
def _launch(
|
||||
x: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
cfg: _Config,
|
||||
) -> None:
|
||||
M, K = x.shape
|
||||
N = weight.shape[0]
|
||||
opts = dict(
|
||||
num_warps=cfg.num_warps,
|
||||
num_stages=cfg.num_stages,
|
||||
matrix_instr_nonkdim=cfg.matrix_instr_nonkdim,
|
||||
waves_per_eu=cfg.waves_per_eu,
|
||||
)
|
||||
grid_m, grid_n = triton.cdiv(M, cfg.block_m), triton.cdiv(N, cfg.block_n)
|
||||
tiles = grid_m * grid_n
|
||||
step = cfg.block_k * cfg.k_split if cfg.packed else cfg.block_k
|
||||
splits = cfg.fused_splits if cfg.fused_splits > 1 or cfg.packed else cfg.k_split
|
||||
k_per_split, splits = _k_partition(K, splits, step)
|
||||
counters = None
|
||||
if cfg.fused_splits > 1 and splits > 1 and tiles <= _SPLIT_COUNTERS:
|
||||
counters = _split_counters(out.device)
|
||||
if counters is not None:
|
||||
fused = splits
|
||||
target = out
|
||||
slots = torch.empty(
|
||||
tiles * fused * cfg.block_m * cfg.block_n,
|
||||
dtype=torch.float32,
|
||||
device=out.device,
|
||||
)
|
||||
grid: tuple[int, ...] = (triton.cdiv(tiles, _XCD_STRIDE) * _XCD_STRIDE * fused,)
|
||||
else:
|
||||
# Without counters, split K through a second launch.
|
||||
fused = 1
|
||||
target = (
|
||||
out
|
||||
if splits == 1
|
||||
else torch.empty((splits, M, N), dtype=torch.float32, device=x.device)
|
||||
)
|
||||
slots = counters = target # Unused by the kernels.
|
||||
if cfg.packed or not cfg.n_first:
|
||||
grid = (grid_m, grid_n, splits)
|
||||
else:
|
||||
grid = (grid_n, grid_m, splits)
|
||||
args = (x, x_scale, weight, weight_scale, target, slots, counters, M, N, K)
|
||||
stride = out.stride(0) if splits == 1 or fused > 1 else N
|
||||
if cfg.packed:
|
||||
_block32_packed_kernel[grid](
|
||||
*args,
|
||||
stride,
|
||||
BLOCK_M=cfg.block_m,
|
||||
BLOCK_N=cfg.block_n,
|
||||
BLOCK_K=cfg.block_k,
|
||||
K_PACK=cfg.k_split,
|
||||
K_PER_SPLIT=k_per_split,
|
||||
EVEN_K=K % step == 0,
|
||||
FUSED_SPLITS=fused,
|
||||
**opts,
|
||||
)
|
||||
else:
|
||||
_block32_tiled_kernel[grid](
|
||||
*args,
|
||||
stride,
|
||||
BLOCK_M=cfg.block_m,
|
||||
BLOCK_N=cfg.block_n,
|
||||
BLOCK_K=cfg.block_k,
|
||||
K_PER_SPLIT=k_per_split,
|
||||
N_FIRST=cfg.n_first,
|
||||
EVEN_K=K % k_per_split == 0,
|
||||
FUSED_SPLITS=fused,
|
||||
**opts,
|
||||
)
|
||||
if fused == 1 and splits > 1:
|
||||
_block32_splitk_reduce_kernel[(triton.cdiv(M, 32), triton.cdiv(N, 64))](
|
||||
target,
|
||||
out,
|
||||
M,
|
||||
N,
|
||||
out.stride(0),
|
||||
SPLIT_K=splits,
|
||||
BLOCK_M=32,
|
||||
BLOCK_N=64,
|
||||
)
|
||||
|
||||
|
||||
def _rocm_mxfp8_block32_gemm_impl(
|
||||
x: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
M, K = x.shape
|
||||
out = torch.empty((M, weight.shape[0]), dtype=out_dtype, device=x.device)
|
||||
if M > 0:
|
||||
_launch(x, x_scale, weight, weight_scale, out, _config(M, weight.shape[0], K))
|
||||
return out
|
||||
|
||||
|
||||
def _rocm_mxfp8_block32_gemm_fake(
|
||||
x: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
return x.new_empty((x.shape[0], weight.shape[0]), dtype=out_dtype)
|
||||
|
||||
|
||||
# An opaque op: the kernel and tile choice branch on the token count, which a
|
||||
# compiled caller must not freeze at its trace-time value.
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_mxfp8_block32_gemm",
|
||||
op_func=_rocm_mxfp8_block32_gemm_impl,
|
||||
fake_impl=_rocm_mxfp8_block32_gemm_fake,
|
||||
)
|
||||
|
||||
|
||||
def rocm_mxfp8_block32_gemm(
|
||||
x: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""``x @ weight.T`` for MXFP8 ``x`` and a 32x32 block-scaled MXFP8 weight.
|
||||
|
||||
Args:
|
||||
x: [M, K] e4m3 activation, contiguous.
|
||||
x_scale: [M, K / 32] E8M0 (uint8) activation scales.
|
||||
weight: [N, K] e4m3 weight, contiguous.
|
||||
weight_scale: [ceil(N / 32), K / 32] E8M0 (uint8) weight block scales.
|
||||
out_dtype: Output dtype.
|
||||
|
||||
Returns:
|
||||
The [M, N] product.
|
||||
|
||||
"""
|
||||
return torch.ops.vllm.rocm_mxfp8_block32_gemm(
|
||||
x, x_scale, weight, weight_scale, out_dtype
|
||||
)
|
||||
@@ -1,295 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Native MXFP8 linear GEMM for AMD CDNA4 (gfx950) via Triton ``tl.dot_scaled``.
|
||||
|
||||
Consumes the FP8 E4M3 weights + E8M0 block scales directly (no dequant-to-BF16);
|
||||
activations are MXFP8-quantized per token. Uses the CDNA4 hardware microscaling
|
||||
matrix cores. ``dot_scaled`` tiles K by 128; a weight whose K is not a multiple
|
||||
of that is dequantized to BF16 once at load and served by a plain linear
|
||||
instead, since ``can_implement`` does not filter on K.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
dequant_mxfp8_to_bf16,
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
from .rocm_block32_gemm import BLOCK_ROWS, rocm_mxfp8_block32_gemm
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mxfp8_linear_kernel(
|
||||
x_ptr,
|
||||
xs_ptr,
|
||||
w_ptr,
|
||||
ws_ptr,
|
||||
out_ptr,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
stride_xm,
|
||||
stride_xk,
|
||||
stride_xsm,
|
||||
stride_xsk,
|
||||
stride_wn,
|
||||
stride_wk,
|
||||
stride_wsn,
|
||||
stride_wsk,
|
||||
stride_om,
|
||||
stride_on,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0)
|
||||
pid_n = tl.program_id(1)
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
offs_k = tl.arange(0, BLOCK_K)
|
||||
offs_sk = tl.arange(0, BLOCK_K // 32)
|
||||
m_mask = offs_m < M
|
||||
n_mask = offs_n < N
|
||||
|
||||
x_ptrs = x_ptr + offs_m[:, None] * stride_xm + offs_k[None, :] * stride_xk
|
||||
xs_ptrs = xs_ptr + offs_m[:, None] * stride_xsm + offs_sk[None, :] * stride_xsk
|
||||
w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk
|
||||
ws_ptrs = ws_ptr + offs_n[:, None] * stride_wsn + offs_sk[None, :] * stride_wsk
|
||||
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for _ in range(0, tl.cdiv(K, BLOCK_K)):
|
||||
x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0)
|
||||
w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
|
||||
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0)
|
||||
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0)
|
||||
acc += tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3")
|
||||
x_ptrs += BLOCK_K * stride_xk
|
||||
w_ptrs += BLOCK_K * stride_wk
|
||||
xs_ptrs += (BLOCK_K // 32) * stride_xsk
|
||||
ws_ptrs += (BLOCK_K // 32) * stride_wsk
|
||||
|
||||
o_ptrs = out_ptr + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on
|
||||
tl.store(
|
||||
o_ptrs, acc.to(out_ptr.dtype.element_ty), mask=m_mask[:, None] & n_mask[None, :]
|
||||
)
|
||||
|
||||
|
||||
def _mxfp8_dot_scaled_linear(
|
||||
x: torch.Tensor, # [M, K] bf16/fp16
|
||||
w: torch.Tensor, # [N, K] fp8 e4m3
|
||||
w_scale: torch.Tensor, # [N, K//32] uint8 (E8M0)
|
||||
) -> torch.Tensor:
|
||||
M, K = x.shape
|
||||
N = w.shape[0]
|
||||
x_q, x_scale = mxfp8_e4m3_quantize(x)
|
||||
out = torch.empty((M, N), dtype=x.dtype, device=x.device)
|
||||
BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages = _select_cfg(M, N, K)
|
||||
grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N))
|
||||
_mxfp8_linear_kernel[grid](
|
||||
x_q,
|
||||
x_scale,
|
||||
w,
|
||||
w_scale,
|
||||
out,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
x_q.stride(0),
|
||||
x_q.stride(1),
|
||||
x_scale.stride(0),
|
||||
x_scale.stride(1),
|
||||
w.stride(0),
|
||||
w.stride(1),
|
||||
w_scale.stride(0),
|
||||
w_scale.stride(1),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
BLOCK_M=BLOCK_M,
|
||||
BLOCK_N=BLOCK_N,
|
||||
BLOCK_K=BLOCK_K,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
# gfx950's async-copy LDS (default in the triton 3.8 this branch pins) leaves no
|
||||
# room for num_stages=3 at BLOCK_K=256; use 2 there.
|
||||
_BK256_STAGES = 3
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if on_gfx950():
|
||||
_BK256_STAGES = 2
|
||||
|
||||
|
||||
def _select_cfg(M, N, K):
|
||||
"""(BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages) — graph-tuned on gfx950.
|
||||
|
||||
The M-bucketed, shape-adaptive tile selection here is the speedup over the
|
||||
upstream 2-bucket launcher. Tiles are pipelined (num_stages>=2, larger BLOCK_K)
|
||||
and occupancy- and shape-aware: keyed on the LOCAL (M, N, K), so it adapts to the
|
||||
TP-sharded shapes (e.g. MiniMax-M3 TP=4 vs TP=8, where local N and K differ) —
|
||||
large-K prefill uses BLOCK_K=256; short-K (K=768) widens N. BLOCK_K must divide K
|
||||
(the K-loop is unmasked), so every BLOCK_K below is guarded to be K-divisible
|
||||
(served K: 384/768/1024/2048/6144).
|
||||
"""
|
||||
if M <= 64:
|
||||
# decode (M in {1,32,64}): tiny-M GEMV is weight-BW + GPU-OCCUPANCY bound. The
|
||||
# lever is NARROW BLOCK_N=16 (maximize N-tiles so more CUs stream the weight in
|
||||
# parallel) + LARGE BLOCK_K (fewer K-iters, bigger coalesced weight loads).
|
||||
# Tuned by CUDA-graph replay latency. Optimal at both TP=4 and TP=8.
|
||||
if K % 1024 == 0: # K=2048, 6144 -> graph-best 16x16x1024 (all M)
|
||||
return 16, 16, 1024, 2, 2
|
||||
if K % 512 == 0:
|
||||
return 16, 16, 512, 2, 3
|
||||
if K % 256 == 0: # K=768 (shared_down) -> graph-best 16x32x256
|
||||
return 16, 32, 256, 4, 3
|
||||
return 16, 32, 128, 4, 3
|
||||
# mid-M (65..256) on SMALL local-N: still occupancy-bound (a 64x64 tile makes too
|
||||
# few N-tiles), so the narrow-BLOCK_N decode-style tile fills the CUs better.
|
||||
# N<=1536 covers the real fused-qkv local N at TP=8: q heads shard but the GQA KV
|
||||
# (4) + sparse-indexer (4) heads are < TP=8, so vLLM replicates them to 1/rank ->
|
||||
# N = 1024 + 4*128 = 1536 (not 2560/2=1280). For the wider 1280<N<=1536 band the
|
||||
# narrow tile only wins up to M=128 (at M=256 the 64x64 tile is better), so cap it
|
||||
# there; N<=1280 keeps the narrow tile through M=256. TP=4 qkv N=2560 is unchanged.
|
||||
if (M <= 256 and N <= 1280) or (M <= 128 and N <= 1536):
|
||||
if K % 1024 == 0:
|
||||
return 16, 16, 1024, 2, 2
|
||||
if K % 512 == 0:
|
||||
return 16, 16, 512, 2, 3
|
||||
if K % 256 == 0:
|
||||
return 16, 16, 256, 2, 3
|
||||
return 16, 16, 128, 2, 3
|
||||
# right-sized launch grid (host-side), used to gate the tall 256-BLOCK_M tile.
|
||||
occ = triton.cdiv(M, 256) * triton.cdiv(N, 128)
|
||||
if K <= 1024: # short-K (shared_down K=384/768; TP=8 o_proj K=1024)
|
||||
if M <= 256:
|
||||
return (64, 64, 256, 8, 2) if K % 256 == 0 else (64, 64, 128, 8, 2)
|
||||
# large prefill. (The former 256x128x256 tall tile was faster only on triton
|
||||
# 3.6; on triton 3.7 its large BLOCK_M register/LDS footprint spills or hits
|
||||
# "out of resources", so use 128x128x256 -- within the known-good footprint.)
|
||||
if M >= 4096 and K >= 1024 and K % 256 == 0 and occ >= 256:
|
||||
return 128, 128, 256, 8, _BK256_STAGES
|
||||
return 128, 128, 128, 8, 3
|
||||
# large-K (K >= 2048). BLOCK_K is K-divisibility-guarded (the K-loop is unmasked):
|
||||
# served large-K is 2048/6144 (%256==0), but fall back to 128 (always divides, since
|
||||
# the entry requires K%128==0) for any other K to stay correct.
|
||||
if M <= 256: # conc~128 decode + small prefill chunk: occupancy tile
|
||||
if K % 512 == 0:
|
||||
return 64, 64, 512, 8, 2
|
||||
return (64, 64, 256, 8, 2) if K % 256 == 0 else (64, 64, 128, 8, 2)
|
||||
if M <= 1024: # medium prefill chunk: BN=64 keeps small-N occupied
|
||||
return (128, 64, 256, 8, 3) if K % 256 == 0 else (128, 64, 128, 8, 3)
|
||||
# large prefill (M > 1024). The previously graph-tuned tall 256x128x256 and deep
|
||||
# 128x128x512 tiles won only on triton 3.6; on triton 3.7 their larger BLOCK_M /
|
||||
# BLOCK_K register+LDS footprint spills (or hits "out of resources" on stricter
|
||||
# ROCm/triton builds). The 128x128x256 tile is equal-or-faster on triton 3.7 (the
|
||||
# M=4096,N=2048,K=6144 shape: ~104us vs the 256x128x256 tile's ~184us), within ~5%
|
||||
# on 3.6, and inside the footprint of the tiles used elsewhere in this selector.
|
||||
# Covers the qkv-class local N=1536 (TP=8 qkv / TP=4 shared_gate_up) and the deep-K
|
||||
# / very-large-M shapes.
|
||||
if K % 256 == 0 and (1280 < N <= 1536 or (occ >= 128 and (K >= 4096 or M >= 4096))):
|
||||
return 128, 128, 256, 8, _BK256_STAGES
|
||||
# small local-N (e.g. TP=8 shared_gate_up N=768): a 64-wide BLOCK_N doubles the
|
||||
# N-tile count -> better CU fill than 128x128 at this mid-large M (~1.4x there).
|
||||
if N <= 1024 and K % 256 == 0:
|
||||
return 128, 64, 256, 8, 3
|
||||
return (128, 128, 256, 8, 2) if K % 256 == 0 else (128, 256, 128, 8, 3)
|
||||
|
||||
|
||||
_DOT_SCALED_K_ALIGN = 128
|
||||
|
||||
|
||||
def _as_block32_scale(weight_scale: torch.Tensor) -> torch.Tensor | None:
|
||||
"""The [N / 32, K / 32] block scales, if every 32 rows share their scales."""
|
||||
N = weight_scale.shape[0]
|
||||
if N % BLOCK_ROWS != 0:
|
||||
return None
|
||||
blocks = weight_scale.view(N // BLOCK_ROWS, BLOCK_ROWS, -1)
|
||||
if not torch.equal(blocks, blocks[:, :1].expand_as(blocks)):
|
||||
return None
|
||||
return blocks[:, 0].contiguous()
|
||||
|
||||
|
||||
class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``."""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_rocm():
|
||||
return False, "not ROCm"
|
||||
# supports_mx() == gfx95x (CDNA4 native microscaling hardware). On other
|
||||
# archs dot_scaled would upcast to BF16, so the kernel selector falls
|
||||
# through to the BF16 emulation (hipBLASLt) path instead.
|
||||
if not current_platform.supports_mx():
|
||||
return False, "native MX requires CDNA4 (gfx95x)"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
weight = layer.weight.data # [N, K] fp8
|
||||
N, K = weight.shape
|
||||
scale_k = K // MXFP8_BLOCK_SIZE
|
||||
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
|
||||
# rocm_mxfp8_block32_gemm's split-K varies with M.
|
||||
block_scale = (
|
||||
None if envs.VLLM_BATCH_INVARIANT else _as_block32_scale(weight_scale)
|
||||
)
|
||||
if block_scale is not None:
|
||||
# Checkpoints with 32x32 scale blocks (DeepSeek V4/V4.1) get them
|
||||
# back from the loader expanded per row; keep the blocks and use
|
||||
# the block-scaled GEMM, which also handles any K % 32 == 0.
|
||||
weight_scale = block_scale
|
||||
elif K % _DOT_SCALED_K_ALIGN != 0:
|
||||
weight = dequant_mxfp8_to_bf16(weight.contiguous(), weight_scale)
|
||||
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
|
||||
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if layer.weight_scale.dtype != MXFP8_SCALE_DTYPE:
|
||||
raise ValueError(
|
||||
f"Expected {MXFP8_SCALE_DTYPE} weight_scale, got "
|
||||
f"{layer.weight_scale.dtype}."
|
||||
)
|
||||
out_shape = (*x.shape[:-1], layer.weight.shape[0])
|
||||
x2d = x.reshape(-1, x.shape[-1])
|
||||
if layer.weight_scale.shape[0] != layer.weight.shape[0]:
|
||||
# One scale row per 32 weight rows (see process_weights_after_loading).
|
||||
x_q, x_scale = mxfp8_e4m3_quantize(x2d)
|
||||
out = rocm_mxfp8_block32_gemm(
|
||||
x_q, x_scale, layer.weight, layer.weight_scale, x.dtype
|
||||
)
|
||||
elif layer.weight.element_size() >= 2:
|
||||
out = torch.nn.functional.linear(x2d, layer.weight.to(x.dtype))
|
||||
elif x2d.shape[-1] % _DOT_SCALED_K_ALIGN == 0:
|
||||
out = _mxfp8_dot_scaled_linear(x2d, layer.weight, layer.weight_scale)
|
||||
else:
|
||||
# process_weights_after_loading dequantizes these weights once, so
|
||||
# this runs only if it did not; dot_scaled would be invalid here.
|
||||
w_bf16 = dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale)
|
||||
out = torch.nn.functional.linear(x2d, w_bf16).to(x.dtype)
|
||||
out = out.reshape(out_shape)
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
return out
|
||||
Reference in New Issue
Block a user