[Mini-VLLM] Remove the ROCm device path, debug routes and arch-gated tests

- Delete csrc/rocm, hipify, the _rocm_C extension and the setup.py rocm
  target path; drop the rocm requirement files
- Delete the aiter, rdna3, skinny-gemm and ROCm fused-MoE kernel modules
  and retarget the linear kernel registries away from them
- Reduce platforms/rocm.py to the GCN architecture predicates that live
  code still imports, and delete the RocmPlatform class and plugin
- Drop the VLLM_SERVER_DEV_MODE debug surface: reset, collective_rpc,
  server_info and sleep routes plus the log_error_stack flag
- Delete lm-format-enforcer, lark and otel-semantic-conventions-ai pins;
  the live server process maps none of them
- Delete tests that only build architectures this fork no longer
  registers, and test directories left without any test file
- Restore the tests/v1/kv_connector mock helpers that scheduler tests
  import at module level

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 19:31:07 +07:00
co-authored by Hinata
parent 2066d3143f
commit 5f3c7bd8ab
162 changed files with 599 additions and 27490 deletions
@@ -1,163 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm BF16x3 MoE router GEMM for fp32 gate weights.
Gates whose weights ship in fp32 (MiniMax-M2/M3, HunYuan-V3) otherwise land in
the ``F.linear`` fallback, which casts the bf16 activations up to fp32 and runs
the GEMM on the fp32 MFMA pipe. Splitting the weight into three bf16 terms
whose sum reproduces it exactly moves the GEMM onto the bf16 pipe and is more
accurate than the path it replaces, which rounds the activations on the way to
fp32 while this one leaves them untouched.
The terms are stacked along the expert dimension into one ``(TERMS * E, K)``
weight, so the router costs a single hipBLASLt GEMM plus a reduction over the
``TERMS`` column blocks of its output.
"""
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
__all__ = [
"BF16X3_TERMS",
"MIN_TOKENS",
"bf16x3_router_gemm",
"is_supported",
"platform_supported",
"split_bf16x3",
]
BF16X3_TERMS = 3
# Below this the fp32 fallback is faster: the GEMM plus reduction has a ~32us
# floor from launch and allocation that small batches cannot amortize.
# Measured on gfx950; the crossover is 1536 at (K=6144, E=128) and (K=3072,
# E=256), and this leaves margin for untested shapes.
MIN_TOKENS = 2048
_MAX_INT32 = (1 << 31) - 1
def split_bf16x3(weight: torch.Tensor) -> torch.Tensor:
"""Split an fp32 weight into bf16 terms whose sum reproduces it exactly.
Returns a contiguous ``(BF16X3_TERMS, E, K)`` tensor. Three
round-to-nearest bf16 terms carry 24 mantissa bits, matching fp32's 24, so
the reconstruction is bit-exact for finite normals down to ``2**-108``.
Raises ValueError on non-finite input: ``inf`` rounds to bf16 ``inf`` and
``inf - inf`` is NaN, which would poison every logit rather than just the
affected experts.
"""
if weight.dtype != torch.float32:
raise ValueError(f"expected fp32 weight, got {weight.dtype}")
if not torch.isfinite(weight).all():
raise ValueError("router weight contains inf/NaN")
terms = []
resid = weight.detach().clone()
for _ in range(BF16X3_TERMS):
hi = resid.to(torch.bfloat16)
terms.append(hi)
resid -= hi.to(torch.float32)
return torch.stack(terms).contiguous()
def platform_supported() -> bool:
"""Whether this platform has a benchmarked bf16x3 router path.
gfx942 has a different fp32:bf16 MFMA ratio and is not yet evaluated.
"""
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
def is_supported(x: torch.Tensor, weight: torch.Tensor) -> bool:
"""Whether this path can serve ``x @ weight.T`` -> fp32 profitably."""
if x.dim() != 2 or weight.dim() != 2:
return False
n, k = weight.shape
m = x.shape[0]
return (
m >= MIN_TOKENS
and x.dtype == torch.bfloat16
and weight.dtype == torch.float32
and x.shape[1] == k
# The weight may be offloaded while the activations are not; the
# fallback this guards would then fail on a device mismatch.
and x.device == weight.device
# torch.mm accepts any stride, but a non-K-contiguous activation costs
# a hidden copy that gives back the win over the fallback.
and x.stride(1) == 1
and 0 < m * BF16X3_TERMS * n <= _MAX_INT32
)
@triton.jit
def _reduce_terms_kernel(
y_ptr,
out_ptr,
M,
stride_ym,
stride_om,
N: tl.constexpr,
TERMS: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
):
pid = tl.program_id(0)
offs_m = pid * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = tl.arange(0, BLOCK_N)
mask = (offs_m < M)[:, None] & (offs_n < N)[None, :]
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for t in tl.static_range(TERMS):
acc += tl.load(
y_ptr + offs_m[:, None] * stride_ym + t * N + offs_n[None, :],
mask=mask,
other=0.0,
)
tl.store(out_ptr + offs_m[:, None] * stride_om + offs_n[None, :], acc, mask=mask)
def bf16x3_router_gemm(
x: torch.Tensor,
weight_split: torch.Tensor,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Router logits in fp32 from bf16 activations and a bf16x3 split weight."""
m, k = x.shape
terms, n, k_w = weight_split.shape
assert terms == BF16X3_TERMS, f"expected {BF16X3_TERMS} terms, got {terms}"
assert k == k_w, f"K mismatch: x has {k}, weight_split has {k_w}"
assert weight_split.dtype == torch.bfloat16, weight_split.dtype
assert weight_split.is_contiguous(), "weight_split must be contiguous"
assert x.dtype == torch.bfloat16, x.dtype
if out is None:
out = torch.empty((m, n), dtype=torch.float32, device=x.device)
else:
assert out.shape == (m, n) and out.is_contiguous(), "bad out layout"
# Rows [t*E, (t+1)*E) of the (TERMS * E, K) view hold term t, so the GEMM
# emits one column block per term and the reduction sums them.
y = torch.empty((m, terms * n), dtype=torch.float32, device=x.device)
torch.mm(x, weight_split.view(terms * n, k).T, out_dtype=torch.float32, out=y)
# 64 rows/program keeps the accumulator small enough to stay in registers
# at the largest supported expert count.
block_m = 64
_reduce_terms_kernel[(triton.cdiv(m, block_m),)](
y,
out,
m,
y.stride(0),
out.stride(0),
N=n,
TERMS=terms,
BLOCK_M=block_m,
BLOCK_N=triton.next_power_of_2(n),
num_warps=4,
)
return out
@@ -29,11 +29,9 @@ class GateLinear(ReplicatedLinear):
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
any shape)
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
5. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
M>=2048)
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype); gfx950
5. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype); gfx950
routes low M through the tier 2 kernel
7. F.linear via ReplicatedLinear (ultimate fallback)
6. F.linear via ReplicatedLinear (ultimate fallback)
The ``out_dtype`` attribute is mutable and can be set after init
(e.g. when the required dtype depends on the expert quantization
@@ -156,34 +154,6 @@ class GateLinear(ReplicatedLinear):
and self.out_dtype == torch.float32
)
# ROCm bf16x3 router GEMM eligibility. out_dtype may still be None here
# and is folded in by set_out_dtype; the split itself is only built
# once the tier is known to be on (see _GateLinearMethod).
from vllm.model_executor.layers.fused_moe.router import (
bf16x3_router_gemm_rocm,
)
self._rocm_bf16x3_weight_eligible = (
self._router_gemm_no_bias
and not batch_invariant
and self.weight.dtype == torch.float32
and bf16x3_router_gemm_rocm.platform_supported()
)
self.allow_rocm_bf16x3_router_gemm = (
self._rocm_bf16x3_weight_eligible and self.out_dtype == torch.float32
)
# Filled in by process_weights_after_loading. Not persistent: it is
# derived from `weight` rather than loaded from a checkpoint.
self.register_buffer("_bf16x3_weight", None, persistent=False)
if self._rocm_bf16x3_weight_eligible:
# Safe to swap after create_weights() because
# UnquantizedLinearMethod holds no per-layer state. Asserted
# because the swap would silently drop quantization if a
# quant_config were ever plumbed into this layer.
assert isinstance(self.quant_method, UnquantizedLinearMethod)
self.quant_method = _GateLinearMethod()
logger.info_once("Enabled ROCm BF16x3 router GEMM.")
# cuteDSL ll_bf16_gemm/ll_fp32w_gemm eligibility. Any dims supported, but
# SM90+ required because:
# 1. PDL support. Both dot-product and split-K kernels.
@@ -246,11 +216,6 @@ class GateLinear(ReplicatedLinear):
):
self.allow_cublas_router_gemm = self.weight.dtype == torch.bfloat16
# out_dtype may start as None -> recompute eligibility here
self.allow_rocm_bf16x3_router_gemm = (
self._rocm_bf16x3_weight_eligible and out_dtype == torch.float32
)
if self.allow_specialized_router_gemm:
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
is_available as is_ll_bf16_gemm_available,
@@ -293,22 +258,6 @@ class GateLinear(ReplicatedLinear):
output = ll_bf16_gemm(x, self.weight)
return self._return(output)
# Tier 5: ROCm bf16x3 router GEMM. Checked before tier 2 because both
# tiers claim the same gfx950 shapes and tier 2 returns
# unconditionally; the custom op below picks between them on the
# runtime num_tokens.
if self.allow_rocm_bf16x3_router_gemm and x.dtype == torch.bfloat16:
if self._bf16x3_weight is None:
logger.warning_once(
"ROCm BF16x3 router GEMM is enabled for %s but its weight "
"split is missing; falling back to fp32.",
self.prefix,
)
else:
output = torch.ops.vllm.rocm_bf16x3_router_gemm_dispatch(
x, self.weight, self._bf16x3_weight
)
return self._return(output)
# Tier 2: fp32 specialized kernel (model-specific shapes, M<=32)
# Dispatch is wrapped in a custom op so that torch.compile/CUDA-graph
@@ -370,90 +319,9 @@ class GateLinear(ReplicatedLinear):
return output if not self.return_bias else (output, output_bias)
class _GateLinearMethod(UnquantizedLinearMethod):
"""UnquantizedLinearMethod plus the ROCm bf16x3 weight split.
Building the split here rather than lazily in forward() charges it to the
weights memory pool ahead of KV cache profiling, and rebuilds it on weight
reload. The latter matters because an in-place ``param.data.copy_()``
leaves both ``data_ptr()`` and ``_version`` untouched, so a stale split
cannot be detected from inside forward().
``allow_rocm_bf16x3_router_gemm`` is final by the time this runs: the only
caller of ``set_out_dtype`` does so during model construction.
"""
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# The base CPU branch may delete layer.weight, so it must run first;
# nothing breaks today only because platform_supported() is False on
# CPU.
super().process_weights_after_loading(layer)
if not layer.allow_rocm_bf16x3_router_gemm:
return
try:
from vllm.model_executor.layers.fused_moe.router import (
bf16x3_router_gemm_rocm,
)
layer._bf16x3_weight = bf16x3_router_gemm_rocm.split_bf16x3(layer.weight)
except ValueError as exc:
layer.allow_rocm_bf16x3_router_gemm = False
logger.warning_once(
"Disabling ROCm BF16x3 router GEMM for %s: %s", layer.prefix, exc
)
_FP32_ROUTER_GEMM_MAX_TOKENS = GateLinear.FP32_MAX_TOKENS
def rocm_bf16x3_router_gemm_dispatch_impl(
x: torch.Tensor,
weight: torch.Tensor,
weight_split: torch.Tensor,
) -> torch.Tensor:
"""Run the ROCm bf16x3 router GEMM, or fall back for small batches.
This must be a custom op because our torch.compile integration does not
support runtime dispatching on num_tokens: models carrying fp32 router
weights (MiniMax-M2, HunYuan-V3) call the gate from
``@support_torch_compile`` model code and vLLM drops all Dynamo guards, so
a plain Python branch on ``x.shape[0]`` would be frozen at first trace.
"""
from vllm.model_executor.layers.fused_moe.router import (
bf16x3_router_gemm_rocm,
)
if bf16x3_router_gemm_rocm.is_supported(x, weight):
return bf16x3_router_gemm_rocm.bf16x3_router_gemm(x, weight_split)
# Below MIN_TOKENS, hand back to the low-M gfx950 kernel this tier is
# checked ahead of.
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
can_use_rocm_fp32_router_gemm,
rocm_fp32_router_gemm,
)
x = x.contiguous()
if can_use_rocm_fp32_router_gemm(x, weight):
return rocm_fp32_router_gemm(x, weight)
return torch.nn.functional.linear(x.float(), weight)
def rocm_bf16x3_router_gemm_dispatch_fake(
x: torch.Tensor,
weight: torch.Tensor,
weight_split: torch.Tensor,
) -> torch.Tensor:
return x.new_empty((x.shape[0], weight.shape[0]), dtype=torch.float32)
direct_register_custom_op(
op_name="rocm_bf16x3_router_gemm_dispatch",
op_func=rocm_bf16x3_router_gemm_dispatch_impl,
fake_impl=rocm_bf16x3_router_gemm_dispatch_fake,
)
def fp32_router_gemm_dispatch_impl(
x: torch.Tensor,
weight: torch.Tensor,
@@ -465,19 +333,6 @@ def fp32_router_gemm_dispatch_impl(
This must be wrapped in a custom op because our torch.compile integration
does not support runtime dispatching on num_tokens.
"""
if current_platform.is_rocm():
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
can_use_rocm_fp32_router_gemm,
rocm_fp32_router_gemm,
)
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
x = x.contiguous()
if can_use_rocm_fp32_router_gemm(x, weight):
return rocm_fp32_router_gemm(x, weight)
if weight.dtype == torch.bfloat16:
return torch.mm(x, weight.T, out_dtype=torch.float32)
return torch.nn.functional.linear(x.float(), weight)
max_tokens = _FP32_ROUTER_GEMM_MAX_TOKENS
if (
allow_bf16x3_router_gemm
@@ -1,175 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Low-token router GEMM with FP32 output for gfx950 (FP32 or BF16 weights)."""
import torch
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_gfx950
from vllm.triton_utils import tl, triton
_MAX_TOKENS = 32
# From this many multiply-adds on, F.linear wins on MI355.
_MAX_MACS = 1 << 27
# Above this num_tokens * num_experts, hipBLASLt wins for BF16 weights.
_MAX_BF16_TOKEN_EXPERTS = 8192
@triton.jit
def _rocm_fp32_router_gemm_kernel(
hidden_states_ptr,
router_weight_ptr,
output_ptr,
M,
K: tl.constexpr,
N: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
EVEN_K: tl.constexpr,
):
pid = tl.program_id(0)
pid_m = pid // N
pid_n = pid - pid_m * N
offsets_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offsets_k = tl.arange(0, BLOCK_K)
partials = tl.zeros((BLOCK_M, BLOCK_K), dtype=tl.float32)
for k_start in range(0, K, BLOCK_K):
offsets_k_block = k_start + offsets_k
if EVEN_K:
hidden_states = tl.load(
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
mask=offsets_m[:, None] < M,
other=0.0,
).to(tl.float32)
router_weight = tl.load(router_weight_ptr + pid_n * K + offsets_k_block).to(
tl.float32
)
else:
k_mask = offsets_k_block < K
hidden_states = tl.load(
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
mask=(offsets_m[:, None] < M) & k_mask[None, :],
other=0.0,
).to(tl.float32)
router_weight = tl.load(
router_weight_ptr + pid_n * K + offsets_k_block,
mask=k_mask,
other=0.0,
).to(tl.float32)
partials += hidden_states * router_weight[None, :]
accumulator = tl.sum(partials, axis=1)
tl.store(
output_ptr + offsets_m * N + pid_n,
accumulator,
mask=offsets_m < M,
)
def _launch_config(
num_experts: int, num_tokens: int, bf16_weight: bool
) -> tuple[int, int, int]:
"""Return (BLOCK_M, BLOCK_K, num_warps), swept on MI355 for M<=32.
One program computes BLOCK_M tokens for one expert, so narrow routers keep
BLOCK_M=1 to fill the GPU and wide routers grow BLOCK_M to reuse weights.
"""
if bf16_weight:
return (1 if num_tokens <= 4 else 2), 1024, 2
if num_experts < 64:
return 1, 1024, 4
wide = num_experts > 256
if num_tokens <= 2:
return 1, 512, 2
if num_tokens <= 4:
return (2, 1024, 2) if wide else (2, 2048, 4)
if num_tokens <= 8:
return (4, 512, 2) if wide else (4, 1024, 4)
if num_tokens <= 16:
return (8, 512, 2) if wide else (4, 1024, 4)
if num_experts >= 768:
return 16, 256, 2
return (8, 256, 2) if wide else (4, 512, 2)
def can_use_rocm_fp32_router_gemm(
hidden_states: torch.Tensor,
router_weight: torch.Tensor,
) -> bool:
"""Return whether the tensors can use the gfx950 fast path."""
try:
_validate_inputs(hidden_states, router_weight)
except (RuntimeError, ValueError):
return False
return True
def _validate_inputs(
hidden_states: torch.Tensor,
router_weight: torch.Tensor,
) -> None:
if not current_platform.is_rocm() or not on_gfx950():
raise RuntimeError("rocm_fp32_router_gemm requires ROCm gfx950")
if hidden_states.dim() != 2 or router_weight.dim() != 2:
raise ValueError("hidden_states and router_weight must be 2D tensors")
if hidden_states.dtype not in (torch.bfloat16, torch.float32):
raise ValueError("hidden_states must have dtype bfloat16 or float32")
if router_weight.dtype not in (torch.float32, torch.bfloat16):
raise ValueError("router_weight must have dtype float32 or bfloat16")
if hidden_states.device.type != "cuda" or router_weight.device.type != "cuda":
raise ValueError("hidden_states and router_weight must be GPU tensors")
if hidden_states.device != router_weight.device:
raise ValueError("hidden_states and router_weight must be on the same device")
if not hidden_states.is_contiguous() or not router_weight.is_contiguous():
raise ValueError("hidden_states and router_weight must be contiguous")
if router_weight.shape[1] != hidden_states.shape[1]:
raise ValueError("router_weight and hidden_states hidden sizes differ")
if not 0 <= hidden_states.shape[0] <= _MAX_TOKENS:
raise ValueError(f"num_tokens must be in [0, {_MAX_TOKENS}]")
if hidden_states.shape[0] * router_weight.numel() >= _MAX_MACS:
raise ValueError(
f"num_tokens * num_experts * hidden_size must be < {_MAX_MACS}"
)
if (
router_weight.dtype == torch.bfloat16
and hidden_states.shape[0] * router_weight.shape[0] > _MAX_BF16_TOKEN_EXPERTS
):
raise ValueError(
f"num_tokens * num_experts must be <= {_MAX_BF16_TOKEN_EXPERTS} "
"for bfloat16 weights"
)
def rocm_fp32_router_gemm(
hidden_states: torch.Tensor,
router_weight: torch.Tensor,
) -> torch.Tensor:
"""Compute ``hidden_states @ router_weight.T`` with FP32 accumulation."""
_validate_inputs(hidden_states, router_weight)
num_tokens = hidden_states.shape[0]
hidden_size = hidden_states.shape[1]
num_experts = router_weight.shape[0]
output = hidden_states.new_empty((num_tokens, num_experts), dtype=torch.float32)
if num_tokens == 0:
return output
block_m, block_k, num_warps = _launch_config(
num_experts, num_tokens, router_weight.dtype == torch.bfloat16
)
block_k = min(block_k, triton.next_power_of_2(hidden_size))
grid = (triton.cdiv(num_tokens, block_m) * num_experts,)
_rocm_fp32_router_gemm_kernel[grid](
hidden_states,
router_weight,
output,
M=num_tokens,
K=hidden_size,
N=num_experts,
BLOCK_M=block_m,
BLOCK_K=block_k,
EVEN_K=hidden_size % block_k == 0,
num_warps=num_warps,
num_stages=1,
)
return output
+3 -118
View File
@@ -250,46 +250,6 @@ def use_aiter_triton_gemm(n, m, k, dtype):
)
def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, bool]:
"""Pick the K-shard split for wvSplitKrc and say whether the shape fits.
Mirrors wvSplitKrc() in csrc/rocm/skinny_gemms.cu, which is also where the
shard cap is explained. Both must pick the same chunkk or the workspace
check here bounds the wrong k_rnd.
Returns:
The CHUNKK the kernel will dispatch with, and whether the CU budget and
split-K workspace admit the shape at all.
"""
# Next ^2 of n
N_p2 = 1 << (n - 1).bit_length()
# How many of 4 waves in a group can work on same 16 Ms at same time?
# This reduces the Ms each group works on, i.e. increasing the CUs needed.
GrpsShrB = min(N_p2 // 16, 4)
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile), and each
# working on a 512-shard of K, how many CUs would we need?
CuNeeded = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512) * GrpsShrB
CHUNKK2_MAX_SHARDS = 11
shards_chunkk2 = (k + 256 - 1) // 256 # 256-wide shards
chunkk = (
2
if (
N_p2 != 16
and CuNeeded * 2 <= cu_count
and shards_chunkk2 <= CHUNKK2_MAX_SHARDS
)
else 1
)
# Deterministic reduction stores one fp32 partial per (M, N, k-shard); all
# of them must fit the split-K workspace.
k_rnd = (k + 512 // chunkk - 1) // (512 // chunkk)
fits = N_p2 * m * k_rnd <= 128 * 1024 * 12 and CuNeeded <= cu_count
return chunkk, fits
# On gfx11 a weight row stride that is a multiple of 2048 B makes the multi-row global
# loads of a GEMM collide on the L2/MALL channel hash. Offsetting the stride by one
# 128 B cache line dodges the collision and keeps each row cache-line aligned.
@@ -338,88 +298,13 @@ def rocm_unquantized_gemm_impl(
return linear_batch_invariant(x, weight, bias)
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx11, on_gfx950
n = x.numel() // x.size(-1)
m = weight.shape[0]
k = weight.shape[1]
cu_count = num_compute_units()
_, fits_wvsplitkrc = wvsplitkrc_dispatch(n, k, m, cu_count)
skinny_operands_compatible = weight.is_contiguous() and (
bias is None or bias.is_contiguous()
)
use_skinny_reduce_counting = (
envs.VLLM_ROCM_USE_SKINNY_GEMM
and on_gfx950()
and x.dtype in [torch.float16, torch.bfloat16]
and x.dim() == 2
and (
10 <= n <= 128
and k % 8 == 0
and k > 512
and m % 16 == 0
and fits_wvsplitkrc
and skinny_operands_compatible
)
)
if use_skinny_reduce_counting:
x_view = x.reshape(-1, x.size(-1)).contiguous()
return ops.wvSplitKrc(x_view, weight, cu_count, bias)
if use_aiter_triton_gemm(n, m, k, x.dtype):
from aiter.ops.triton.gemm_a16w16 import gemm_a16w16
return gemm_a16w16(x, weight, bias)
skinny_gemm_enabled = (
envs.VLLM_ROCM_USE_SKINNY_GEMM
and (on_gfx9() or on_gfx1x())
# build (gfx9/gfx11 ISA); fall back to torch GEMM there.
# TODO GFX1250: Include once skinny GEMM is supported on gfx1250
and x.dtype in [torch.float16, torch.bfloat16]
and k % 8 == 0
)
# wvSplitK allows row strides. Perf impact is only validated on gfx11.
weight_ok_for_wvsplitk = weight.is_contiguous() or (
on_gfx11() and weight.stride(1) == 1
)
use_wvsplitk = (
skinny_gemm_enabled
and weight_ok_for_wvsplitk
and (bias is None or bias.is_contiguous())
and (m == 1 or m > 8)
and 0 < n <= 5
)
use_llmm1 = (
skinny_gemm_enabled
and weight.is_contiguous()
and bias is None
and m % 4 == 0
and n == 1
and k <= 8192
)
if use_wvsplitk:
# The skinny kernels assume contiguous K elements. A shape-preserving
# reshape can retain a transposed activation's non-contiguous strides.
# Note: Only build that view inside the branches that consume it.
x_view = x.reshape(-1, x.size(-1)).contiguous()
out = ops.wvSplitK(weight, x_view, cu_count, bias)
return out.reshape(*x.shape[:-1], weight.shape[0])
if use_llmm1:
x_view = x.reshape(-1, x.size(-1)).contiguous()
out = ops.LLMM1(weight, x_view, 4)
return out.reshape(*x.shape[:-1], weight.shape[0])
if rocm_aiter_ops.is_tgemm_enabled():
from aiter.tuned_gemm import tgemm
@@ -462,11 +347,11 @@ def warmup_rocm_skinny_gemm_workspaces(device: torch.device) -> None:
"""Eagerly allocate wvSplitKrc's per-device split-K workspace pool.
wvSplitKrc partitions one per-device allocation into ``kWvSlots`` slots
(csrc/rocm/skinny_gemms.cu) and hands each stream one on first use, so that
and hands each stream one on first use, so that
two streams never share the split-K partials and counters.
The pool is otherwise created lazily on the first qualifying GEMM
(csrc/rocm/skinny_gemms.cu), which can be the first real request — after
The pool is otherwise created lazily on the first qualifying GEMM,
which can be the first real request — after
the KV cache backing buffer exists. If it landed in that segment's rounding
tail, it would pin the entire segment at engine shutdown; it could also land
inside a cudagraph capture, where it would be taken from the graph's private