[Mini-VLLM] Remove the ROCm device path, debug routes and arch-gated tests
- Delete csrc/rocm, hipify, the _rocm_C extension and the setup.py rocm target path; drop the rocm requirement files - Delete the aiter, rdna3, skinny-gemm and ROCm fused-MoE kernel modules and retarget the linear kernel registries away from them - Reduce platforms/rocm.py to the GCN architecture predicates that live code still imports, and delete the RocmPlatform class and plugin - Drop the VLLM_SERVER_DEV_MODE debug surface: reset, collective_rpc, server_info and sleep routes plus the log_error_stack flag - Delete lm-format-enforcer, lark and otel-semantic-conventions-ai pins; the live server process maps none of them - Delete tests that only build architectures this fork no longer registers, and test directories left without any test file - Restore the tests/v1/kv_connector mock helpers that scheduler tests import at module level Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,163 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm BF16x3 MoE router GEMM for fp32 gate weights.
|
||||
|
||||
Gates whose weights ship in fp32 (MiniMax-M2/M3, HunYuan-V3) otherwise land in
|
||||
the ``F.linear`` fallback, which casts the bf16 activations up to fp32 and runs
|
||||
the GEMM on the fp32 MFMA pipe. Splitting the weight into three bf16 terms
|
||||
whose sum reproduces it exactly moves the GEMM onto the bf16 pipe and is more
|
||||
accurate than the path it replaces, which rounds the activations on the way to
|
||||
fp32 while this one leaves them untouched.
|
||||
|
||||
The terms are stacked along the expert dimension into one ``(TERMS * E, K)``
|
||||
weight, so the router costs a single hipBLASLt GEMM plus a reduction over the
|
||||
``TERMS`` column blocks of its output.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
__all__ = [
|
||||
"BF16X3_TERMS",
|
||||
"MIN_TOKENS",
|
||||
"bf16x3_router_gemm",
|
||||
"is_supported",
|
||||
"platform_supported",
|
||||
"split_bf16x3",
|
||||
]
|
||||
|
||||
BF16X3_TERMS = 3
|
||||
|
||||
# Below this the fp32 fallback is faster: the GEMM plus reduction has a ~32us
|
||||
# floor from launch and allocation that small batches cannot amortize.
|
||||
# Measured on gfx950; the crossover is 1536 at (K=6144, E=128) and (K=3072,
|
||||
# E=256), and this leaves margin for untested shapes.
|
||||
MIN_TOKENS = 2048
|
||||
|
||||
_MAX_INT32 = (1 << 31) - 1
|
||||
|
||||
|
||||
def split_bf16x3(weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Split an fp32 weight into bf16 terms whose sum reproduces it exactly.
|
||||
|
||||
Returns a contiguous ``(BF16X3_TERMS, E, K)`` tensor. Three
|
||||
round-to-nearest bf16 terms carry 24 mantissa bits, matching fp32's 24, so
|
||||
the reconstruction is bit-exact for finite normals down to ``2**-108``.
|
||||
|
||||
Raises ValueError on non-finite input: ``inf`` rounds to bf16 ``inf`` and
|
||||
``inf - inf`` is NaN, which would poison every logit rather than just the
|
||||
affected experts.
|
||||
"""
|
||||
if weight.dtype != torch.float32:
|
||||
raise ValueError(f"expected fp32 weight, got {weight.dtype}")
|
||||
if not torch.isfinite(weight).all():
|
||||
raise ValueError("router weight contains inf/NaN")
|
||||
terms = []
|
||||
resid = weight.detach().clone()
|
||||
for _ in range(BF16X3_TERMS):
|
||||
hi = resid.to(torch.bfloat16)
|
||||
terms.append(hi)
|
||||
resid -= hi.to(torch.float32)
|
||||
return torch.stack(terms).contiguous()
|
||||
|
||||
|
||||
def platform_supported() -> bool:
|
||||
"""Whether this platform has a benchmarked bf16x3 router path.
|
||||
|
||||
gfx942 has a different fp32:bf16 MFMA ratio and is not yet evaluated.
|
||||
"""
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
def is_supported(x: torch.Tensor, weight: torch.Tensor) -> bool:
|
||||
"""Whether this path can serve ``x @ weight.T`` -> fp32 profitably."""
|
||||
if x.dim() != 2 or weight.dim() != 2:
|
||||
return False
|
||||
n, k = weight.shape
|
||||
m = x.shape[0]
|
||||
return (
|
||||
m >= MIN_TOKENS
|
||||
and x.dtype == torch.bfloat16
|
||||
and weight.dtype == torch.float32
|
||||
and x.shape[1] == k
|
||||
# The weight may be offloaded while the activations are not; the
|
||||
# fallback this guards would then fail on a device mismatch.
|
||||
and x.device == weight.device
|
||||
# torch.mm accepts any stride, but a non-K-contiguous activation costs
|
||||
# a hidden copy that gives back the win over the fallback.
|
||||
and x.stride(1) == 1
|
||||
and 0 < m * BF16X3_TERMS * n <= _MAX_INT32
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _reduce_terms_kernel(
|
||||
y_ptr,
|
||||
out_ptr,
|
||||
M,
|
||||
stride_ym,
|
||||
stride_om,
|
||||
N: tl.constexpr,
|
||||
TERMS: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
offs_m = pid * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = tl.arange(0, BLOCK_N)
|
||||
mask = (offs_m < M)[:, None] & (offs_n < N)[None, :]
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for t in tl.static_range(TERMS):
|
||||
acc += tl.load(
|
||||
y_ptr + offs_m[:, None] * stride_ym + t * N + offs_n[None, :],
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(out_ptr + offs_m[:, None] * stride_om + offs_n[None, :], acc, mask=mask)
|
||||
|
||||
|
||||
def bf16x3_router_gemm(
|
||||
x: torch.Tensor,
|
||||
weight_split: torch.Tensor,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Router logits in fp32 from bf16 activations and a bf16x3 split weight."""
|
||||
m, k = x.shape
|
||||
terms, n, k_w = weight_split.shape
|
||||
assert terms == BF16X3_TERMS, f"expected {BF16X3_TERMS} terms, got {terms}"
|
||||
assert k == k_w, f"K mismatch: x has {k}, weight_split has {k_w}"
|
||||
assert weight_split.dtype == torch.bfloat16, weight_split.dtype
|
||||
assert weight_split.is_contiguous(), "weight_split must be contiguous"
|
||||
assert x.dtype == torch.bfloat16, x.dtype
|
||||
if out is None:
|
||||
out = torch.empty((m, n), dtype=torch.float32, device=x.device)
|
||||
else:
|
||||
assert out.shape == (m, n) and out.is_contiguous(), "bad out layout"
|
||||
|
||||
# Rows [t*E, (t+1)*E) of the (TERMS * E, K) view hold term t, so the GEMM
|
||||
# emits one column block per term and the reduction sums them.
|
||||
y = torch.empty((m, terms * n), dtype=torch.float32, device=x.device)
|
||||
torch.mm(x, weight_split.view(terms * n, k).T, out_dtype=torch.float32, out=y)
|
||||
|
||||
# 64 rows/program keeps the accumulator small enough to stay in registers
|
||||
# at the largest supported expert count.
|
||||
block_m = 64
|
||||
_reduce_terms_kernel[(triton.cdiv(m, block_m),)](
|
||||
y,
|
||||
out,
|
||||
m,
|
||||
y.stride(0),
|
||||
out.stride(0),
|
||||
N=n,
|
||||
TERMS=terms,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_N=triton.next_power_of_2(n),
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
@@ -29,11 +29,9 @@ class GateLinear(ReplicatedLinear):
|
||||
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
|
||||
any shape)
|
||||
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
|
||||
5. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
|
||||
M>=2048)
|
||||
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype); gfx950
|
||||
5. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype); gfx950
|
||||
routes low M through the tier 2 kernel
|
||||
7. F.linear via ReplicatedLinear (ultimate fallback)
|
||||
6. F.linear via ReplicatedLinear (ultimate fallback)
|
||||
|
||||
The ``out_dtype`` attribute is mutable and can be set after init
|
||||
(e.g. when the required dtype depends on the expert quantization
|
||||
@@ -156,34 +154,6 @@ class GateLinear(ReplicatedLinear):
|
||||
and self.out_dtype == torch.float32
|
||||
)
|
||||
|
||||
# ROCm bf16x3 router GEMM eligibility. out_dtype may still be None here
|
||||
# and is folded in by set_out_dtype; the split itself is only built
|
||||
# once the tier is known to be on (see _GateLinearMethod).
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
bf16x3_router_gemm_rocm,
|
||||
)
|
||||
|
||||
self._rocm_bf16x3_weight_eligible = (
|
||||
self._router_gemm_no_bias
|
||||
and not batch_invariant
|
||||
and self.weight.dtype == torch.float32
|
||||
and bf16x3_router_gemm_rocm.platform_supported()
|
||||
)
|
||||
self.allow_rocm_bf16x3_router_gemm = (
|
||||
self._rocm_bf16x3_weight_eligible and self.out_dtype == torch.float32
|
||||
)
|
||||
# Filled in by process_weights_after_loading. Not persistent: it is
|
||||
# derived from `weight` rather than loaded from a checkpoint.
|
||||
self.register_buffer("_bf16x3_weight", None, persistent=False)
|
||||
if self._rocm_bf16x3_weight_eligible:
|
||||
# Safe to swap after create_weights() because
|
||||
# UnquantizedLinearMethod holds no per-layer state. Asserted
|
||||
# because the swap would silently drop quantization if a
|
||||
# quant_config were ever plumbed into this layer.
|
||||
assert isinstance(self.quant_method, UnquantizedLinearMethod)
|
||||
self.quant_method = _GateLinearMethod()
|
||||
logger.info_once("Enabled ROCm BF16x3 router GEMM.")
|
||||
|
||||
# cuteDSL ll_bf16_gemm/ll_fp32w_gemm eligibility. Any dims supported, but
|
||||
# SM90+ required because:
|
||||
# 1. PDL support. Both dot-product and split-K kernels.
|
||||
@@ -246,11 +216,6 @@ class GateLinear(ReplicatedLinear):
|
||||
):
|
||||
self.allow_cublas_router_gemm = self.weight.dtype == torch.bfloat16
|
||||
|
||||
# out_dtype may start as None -> recompute eligibility here
|
||||
self.allow_rocm_bf16x3_router_gemm = (
|
||||
self._rocm_bf16x3_weight_eligible and out_dtype == torch.float32
|
||||
)
|
||||
|
||||
if self.allow_specialized_router_gemm:
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
|
||||
is_available as is_ll_bf16_gemm_available,
|
||||
@@ -293,22 +258,6 @@ class GateLinear(ReplicatedLinear):
|
||||
output = ll_bf16_gemm(x, self.weight)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 5: ROCm bf16x3 router GEMM. Checked before tier 2 because both
|
||||
# tiers claim the same gfx950 shapes and tier 2 returns
|
||||
# unconditionally; the custom op below picks between them on the
|
||||
# runtime num_tokens.
|
||||
if self.allow_rocm_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||
if self._bf16x3_weight is None:
|
||||
logger.warning_once(
|
||||
"ROCm BF16x3 router GEMM is enabled for %s but its weight "
|
||||
"split is missing; falling back to fp32.",
|
||||
self.prefix,
|
||||
)
|
||||
else:
|
||||
output = torch.ops.vllm.rocm_bf16x3_router_gemm_dispatch(
|
||||
x, self.weight, self._bf16x3_weight
|
||||
)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 2: fp32 specialized kernel (model-specific shapes, M<=32)
|
||||
# Dispatch is wrapped in a custom op so that torch.compile/CUDA-graph
|
||||
@@ -370,90 +319,9 @@ class GateLinear(ReplicatedLinear):
|
||||
return output if not self.return_bias else (output, output_bias)
|
||||
|
||||
|
||||
class _GateLinearMethod(UnquantizedLinearMethod):
|
||||
"""UnquantizedLinearMethod plus the ROCm bf16x3 weight split.
|
||||
|
||||
Building the split here rather than lazily in forward() charges it to the
|
||||
weights memory pool ahead of KV cache profiling, and rebuilds it on weight
|
||||
reload. The latter matters because an in-place ``param.data.copy_()``
|
||||
leaves both ``data_ptr()`` and ``_version`` untouched, so a stale split
|
||||
cannot be detected from inside forward().
|
||||
|
||||
``allow_rocm_bf16x3_router_gemm`` is final by the time this runs: the only
|
||||
caller of ``set_out_dtype`` does so during model construction.
|
||||
"""
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# The base CPU branch may delete layer.weight, so it must run first;
|
||||
# nothing breaks today only because platform_supported() is False on
|
||||
# CPU.
|
||||
super().process_weights_after_loading(layer)
|
||||
if not layer.allow_rocm_bf16x3_router_gemm:
|
||||
return
|
||||
try:
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
bf16x3_router_gemm_rocm,
|
||||
)
|
||||
|
||||
layer._bf16x3_weight = bf16x3_router_gemm_rocm.split_bf16x3(layer.weight)
|
||||
except ValueError as exc:
|
||||
layer.allow_rocm_bf16x3_router_gemm = False
|
||||
logger.warning_once(
|
||||
"Disabling ROCm BF16x3 router GEMM for %s: %s", layer.prefix, exc
|
||||
)
|
||||
|
||||
|
||||
_FP32_ROUTER_GEMM_MAX_TOKENS = GateLinear.FP32_MAX_TOKENS
|
||||
|
||||
|
||||
def rocm_bf16x3_router_gemm_dispatch_impl(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_split: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Run the ROCm bf16x3 router GEMM, or fall back for small batches.
|
||||
|
||||
This must be a custom op because our torch.compile integration does not
|
||||
support runtime dispatching on num_tokens: models carrying fp32 router
|
||||
weights (MiniMax-M2, HunYuan-V3) call the gate from
|
||||
``@support_torch_compile`` model code and vLLM drops all Dynamo guards, so
|
||||
a plain Python branch on ``x.shape[0]`` would be frozen at first trace.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
bf16x3_router_gemm_rocm,
|
||||
)
|
||||
|
||||
if bf16x3_router_gemm_rocm.is_supported(x, weight):
|
||||
return bf16x3_router_gemm_rocm.bf16x3_router_gemm(x, weight_split)
|
||||
# Below MIN_TOKENS, hand back to the low-M gfx950 kernel this tier is
|
||||
# checked ahead of.
|
||||
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
|
||||
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
|
||||
can_use_rocm_fp32_router_gemm,
|
||||
rocm_fp32_router_gemm,
|
||||
)
|
||||
|
||||
x = x.contiguous()
|
||||
if can_use_rocm_fp32_router_gemm(x, weight):
|
||||
return rocm_fp32_router_gemm(x, weight)
|
||||
return torch.nn.functional.linear(x.float(), weight)
|
||||
|
||||
|
||||
def rocm_bf16x3_router_gemm_dispatch_fake(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_split: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return x.new_empty((x.shape[0], weight.shape[0]), dtype=torch.float32)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_bf16x3_router_gemm_dispatch",
|
||||
op_func=rocm_bf16x3_router_gemm_dispatch_impl,
|
||||
fake_impl=rocm_bf16x3_router_gemm_dispatch_fake,
|
||||
)
|
||||
|
||||
|
||||
def fp32_router_gemm_dispatch_impl(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
@@ -465,19 +333,6 @@ def fp32_router_gemm_dispatch_impl(
|
||||
This must be wrapped in a custom op because our torch.compile integration
|
||||
does not support runtime dispatching on num_tokens.
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
|
||||
can_use_rocm_fp32_router_gemm,
|
||||
rocm_fp32_router_gemm,
|
||||
)
|
||||
|
||||
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
|
||||
x = x.contiguous()
|
||||
if can_use_rocm_fp32_router_gemm(x, weight):
|
||||
return rocm_fp32_router_gemm(x, weight)
|
||||
if weight.dtype == torch.bfloat16:
|
||||
return torch.mm(x, weight.T, out_dtype=torch.float32)
|
||||
return torch.nn.functional.linear(x.float(), weight)
|
||||
max_tokens = _FP32_ROUTER_GEMM_MAX_TOKENS
|
||||
if (
|
||||
allow_bf16x3_router_gemm
|
||||
|
||||
@@ -1,175 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Low-token router GEMM with FP32 output for gfx950 (FP32 or BF16 weights)."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
_MAX_TOKENS = 32
|
||||
# From this many multiply-adds on, F.linear wins on MI355.
|
||||
_MAX_MACS = 1 << 27
|
||||
# Above this num_tokens * num_experts, hipBLASLt wins for BF16 weights.
|
||||
_MAX_BF16_TOKEN_EXPERTS = 8192
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rocm_fp32_router_gemm_kernel(
|
||||
hidden_states_ptr,
|
||||
router_weight_ptr,
|
||||
output_ptr,
|
||||
M,
|
||||
K: tl.constexpr,
|
||||
N: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
pid_m = pid // N
|
||||
pid_n = pid - pid_m * N
|
||||
offsets_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offsets_k = tl.arange(0, BLOCK_K)
|
||||
partials = tl.zeros((BLOCK_M, BLOCK_K), dtype=tl.float32)
|
||||
|
||||
for k_start in range(0, K, BLOCK_K):
|
||||
offsets_k_block = k_start + offsets_k
|
||||
if EVEN_K:
|
||||
hidden_states = tl.load(
|
||||
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
|
||||
mask=offsets_m[:, None] < M,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
router_weight = tl.load(router_weight_ptr + pid_n * K + offsets_k_block).to(
|
||||
tl.float32
|
||||
)
|
||||
else:
|
||||
k_mask = offsets_k_block < K
|
||||
hidden_states = tl.load(
|
||||
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
|
||||
mask=(offsets_m[:, None] < M) & k_mask[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
router_weight = tl.load(
|
||||
router_weight_ptr + pid_n * K + offsets_k_block,
|
||||
mask=k_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
partials += hidden_states * router_weight[None, :]
|
||||
|
||||
accumulator = tl.sum(partials, axis=1)
|
||||
tl.store(
|
||||
output_ptr + offsets_m * N + pid_n,
|
||||
accumulator,
|
||||
mask=offsets_m < M,
|
||||
)
|
||||
|
||||
|
||||
def _launch_config(
|
||||
num_experts: int, num_tokens: int, bf16_weight: bool
|
||||
) -> tuple[int, int, int]:
|
||||
"""Return (BLOCK_M, BLOCK_K, num_warps), swept on MI355 for M<=32.
|
||||
|
||||
One program computes BLOCK_M tokens for one expert, so narrow routers keep
|
||||
BLOCK_M=1 to fill the GPU and wide routers grow BLOCK_M to reuse weights.
|
||||
"""
|
||||
if bf16_weight:
|
||||
return (1 if num_tokens <= 4 else 2), 1024, 2
|
||||
if num_experts < 64:
|
||||
return 1, 1024, 4
|
||||
wide = num_experts > 256
|
||||
if num_tokens <= 2:
|
||||
return 1, 512, 2
|
||||
if num_tokens <= 4:
|
||||
return (2, 1024, 2) if wide else (2, 2048, 4)
|
||||
if num_tokens <= 8:
|
||||
return (4, 512, 2) if wide else (4, 1024, 4)
|
||||
if num_tokens <= 16:
|
||||
return (8, 512, 2) if wide else (4, 1024, 4)
|
||||
if num_experts >= 768:
|
||||
return 16, 256, 2
|
||||
return (8, 256, 2) if wide else (4, 512, 2)
|
||||
|
||||
|
||||
def can_use_rocm_fp32_router_gemm(
|
||||
hidden_states: torch.Tensor,
|
||||
router_weight: torch.Tensor,
|
||||
) -> bool:
|
||||
"""Return whether the tensors can use the gfx950 fast path."""
|
||||
try:
|
||||
_validate_inputs(hidden_states, router_weight)
|
||||
except (RuntimeError, ValueError):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _validate_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
router_weight: torch.Tensor,
|
||||
) -> None:
|
||||
if not current_platform.is_rocm() or not on_gfx950():
|
||||
raise RuntimeError("rocm_fp32_router_gemm requires ROCm gfx950")
|
||||
if hidden_states.dim() != 2 or router_weight.dim() != 2:
|
||||
raise ValueError("hidden_states and router_weight must be 2D tensors")
|
||||
if hidden_states.dtype not in (torch.bfloat16, torch.float32):
|
||||
raise ValueError("hidden_states must have dtype bfloat16 or float32")
|
||||
if router_weight.dtype not in (torch.float32, torch.bfloat16):
|
||||
raise ValueError("router_weight must have dtype float32 or bfloat16")
|
||||
if hidden_states.device.type != "cuda" or router_weight.device.type != "cuda":
|
||||
raise ValueError("hidden_states and router_weight must be GPU tensors")
|
||||
if hidden_states.device != router_weight.device:
|
||||
raise ValueError("hidden_states and router_weight must be on the same device")
|
||||
if not hidden_states.is_contiguous() or not router_weight.is_contiguous():
|
||||
raise ValueError("hidden_states and router_weight must be contiguous")
|
||||
if router_weight.shape[1] != hidden_states.shape[1]:
|
||||
raise ValueError("router_weight and hidden_states hidden sizes differ")
|
||||
if not 0 <= hidden_states.shape[0] <= _MAX_TOKENS:
|
||||
raise ValueError(f"num_tokens must be in [0, {_MAX_TOKENS}]")
|
||||
if hidden_states.shape[0] * router_weight.numel() >= _MAX_MACS:
|
||||
raise ValueError(
|
||||
f"num_tokens * num_experts * hidden_size must be < {_MAX_MACS}"
|
||||
)
|
||||
if (
|
||||
router_weight.dtype == torch.bfloat16
|
||||
and hidden_states.shape[0] * router_weight.shape[0] > _MAX_BF16_TOKEN_EXPERTS
|
||||
):
|
||||
raise ValueError(
|
||||
f"num_tokens * num_experts must be <= {_MAX_BF16_TOKEN_EXPERTS} "
|
||||
"for bfloat16 weights"
|
||||
)
|
||||
|
||||
|
||||
def rocm_fp32_router_gemm(
|
||||
hidden_states: torch.Tensor,
|
||||
router_weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Compute ``hidden_states @ router_weight.T`` with FP32 accumulation."""
|
||||
_validate_inputs(hidden_states, router_weight)
|
||||
num_tokens = hidden_states.shape[0]
|
||||
hidden_size = hidden_states.shape[1]
|
||||
num_experts = router_weight.shape[0]
|
||||
output = hidden_states.new_empty((num_tokens, num_experts), dtype=torch.float32)
|
||||
if num_tokens == 0:
|
||||
return output
|
||||
|
||||
block_m, block_k, num_warps = _launch_config(
|
||||
num_experts, num_tokens, router_weight.dtype == torch.bfloat16
|
||||
)
|
||||
block_k = min(block_k, triton.next_power_of_2(hidden_size))
|
||||
grid = (triton.cdiv(num_tokens, block_m) * num_experts,)
|
||||
_rocm_fp32_router_gemm_kernel[grid](
|
||||
hidden_states,
|
||||
router_weight,
|
||||
output,
|
||||
M=num_tokens,
|
||||
K=hidden_size,
|
||||
N=num_experts,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_K=block_k,
|
||||
EVEN_K=hidden_size % block_k == 0,
|
||||
num_warps=num_warps,
|
||||
num_stages=1,
|
||||
)
|
||||
return output
|
||||
@@ -250,46 +250,6 @@ def use_aiter_triton_gemm(n, m, k, dtype):
|
||||
)
|
||||
|
||||
|
||||
def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, bool]:
|
||||
"""Pick the K-shard split for wvSplitKrc and say whether the shape fits.
|
||||
|
||||
Mirrors wvSplitKrc() in csrc/rocm/skinny_gemms.cu, which is also where the
|
||||
shard cap is explained. Both must pick the same chunkk or the workspace
|
||||
check here bounds the wrong k_rnd.
|
||||
|
||||
Returns:
|
||||
The CHUNKK the kernel will dispatch with, and whether the CU budget and
|
||||
split-K workspace admit the shape at all.
|
||||
|
||||
"""
|
||||
# Next ^2 of n
|
||||
N_p2 = 1 << (n - 1).bit_length()
|
||||
# How many of 4 waves in a group can work on same 16 Ms at same time?
|
||||
# This reduces the Ms each group works on, i.e. increasing the CUs needed.
|
||||
GrpsShrB = min(N_p2 // 16, 4)
|
||||
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile), and each
|
||||
# working on a 512-shard of K, how many CUs would we need?
|
||||
CuNeeded = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512) * GrpsShrB
|
||||
|
||||
CHUNKK2_MAX_SHARDS = 11
|
||||
shards_chunkk2 = (k + 256 - 1) // 256 # 256-wide shards
|
||||
chunkk = (
|
||||
2
|
||||
if (
|
||||
N_p2 != 16
|
||||
and CuNeeded * 2 <= cu_count
|
||||
and shards_chunkk2 <= CHUNKK2_MAX_SHARDS
|
||||
)
|
||||
else 1
|
||||
)
|
||||
|
||||
# Deterministic reduction stores one fp32 partial per (M, N, k-shard); all
|
||||
# of them must fit the split-K workspace.
|
||||
k_rnd = (k + 512 // chunkk - 1) // (512 // chunkk)
|
||||
fits = N_p2 * m * k_rnd <= 128 * 1024 * 12 and CuNeeded <= cu_count
|
||||
return chunkk, fits
|
||||
|
||||
|
||||
# On gfx11 a weight row stride that is a multiple of 2048 B makes the multi-row global
|
||||
# loads of a GEMM collide on the L2/MALL channel hash. Offsetting the stride by one
|
||||
# 128 B cache line dodges the collision and keeps each row cache-line aligned.
|
||||
@@ -338,88 +298,13 @@ def rocm_unquantized_gemm_impl(
|
||||
|
||||
return linear_batch_invariant(x, weight, bias)
|
||||
|
||||
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx11, on_gfx950
|
||||
|
||||
n = x.numel() // x.size(-1)
|
||||
m = weight.shape[0]
|
||||
k = weight.shape[1]
|
||||
|
||||
cu_count = num_compute_units()
|
||||
|
||||
_, fits_wvsplitkrc = wvsplitkrc_dispatch(n, k, m, cu_count)
|
||||
|
||||
skinny_operands_compatible = weight.is_contiguous() and (
|
||||
bias is None or bias.is_contiguous()
|
||||
)
|
||||
|
||||
use_skinny_reduce_counting = (
|
||||
envs.VLLM_ROCM_USE_SKINNY_GEMM
|
||||
and on_gfx950()
|
||||
and x.dtype in [torch.float16, torch.bfloat16]
|
||||
and x.dim() == 2
|
||||
and (
|
||||
10 <= n <= 128
|
||||
and k % 8 == 0
|
||||
and k > 512
|
||||
and m % 16 == 0
|
||||
and fits_wvsplitkrc
|
||||
and skinny_operands_compatible
|
||||
)
|
||||
)
|
||||
|
||||
if use_skinny_reduce_counting:
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
return ops.wvSplitKrc(x_view, weight, cu_count, bias)
|
||||
|
||||
if use_aiter_triton_gemm(n, m, k, x.dtype):
|
||||
from aiter.ops.triton.gemm_a16w16 import gemm_a16w16
|
||||
|
||||
return gemm_a16w16(x, weight, bias)
|
||||
|
||||
skinny_gemm_enabled = (
|
||||
envs.VLLM_ROCM_USE_SKINNY_GEMM
|
||||
and (on_gfx9() or on_gfx1x())
|
||||
# build (gfx9/gfx11 ISA); fall back to torch GEMM there.
|
||||
# TODO GFX1250: Include once skinny GEMM is supported on gfx1250
|
||||
and x.dtype in [torch.float16, torch.bfloat16]
|
||||
and k % 8 == 0
|
||||
)
|
||||
|
||||
# wvSplitK allows row strides. Perf impact is only validated on gfx11.
|
||||
weight_ok_for_wvsplitk = weight.is_contiguous() or (
|
||||
on_gfx11() and weight.stride(1) == 1
|
||||
)
|
||||
|
||||
use_wvsplitk = (
|
||||
skinny_gemm_enabled
|
||||
and weight_ok_for_wvsplitk
|
||||
and (bias is None or bias.is_contiguous())
|
||||
and (m == 1 or m > 8)
|
||||
and 0 < n <= 5
|
||||
)
|
||||
|
||||
use_llmm1 = (
|
||||
skinny_gemm_enabled
|
||||
and weight.is_contiguous()
|
||||
and bias is None
|
||||
and m % 4 == 0
|
||||
and n == 1
|
||||
and k <= 8192
|
||||
)
|
||||
|
||||
if use_wvsplitk:
|
||||
# The skinny kernels assume contiguous K elements. A shape-preserving
|
||||
# reshape can retain a transposed activation's non-contiguous strides.
|
||||
# Note: Only build that view inside the branches that consume it.
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
out = ops.wvSplitK(weight, x_view, cu_count, bias)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
|
||||
if use_llmm1:
|
||||
x_view = x.reshape(-1, x.size(-1)).contiguous()
|
||||
out = ops.LLMM1(weight, x_view, 4)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
|
||||
if rocm_aiter_ops.is_tgemm_enabled():
|
||||
from aiter.tuned_gemm import tgemm
|
||||
|
||||
@@ -462,11 +347,11 @@ def warmup_rocm_skinny_gemm_workspaces(device: torch.device) -> None:
|
||||
"""Eagerly allocate wvSplitKrc's per-device split-K workspace pool.
|
||||
|
||||
wvSplitKrc partitions one per-device allocation into ``kWvSlots`` slots
|
||||
(csrc/rocm/skinny_gemms.cu) and hands each stream one on first use, so that
|
||||
and hands each stream one on first use, so that
|
||||
two streams never share the split-K partials and counters.
|
||||
|
||||
The pool is otherwise created lazily on the first qualifying GEMM
|
||||
(csrc/rocm/skinny_gemms.cu), which can be the first real request — after
|
||||
The pool is otherwise created lazily on the first qualifying GEMM,
|
||||
which can be the first real request — after
|
||||
the KV cache backing buffer exists. If it landed in that segment's rounding
|
||||
tail, it would pin the entire segment at engine shutdown; it could also land
|
||||
inside a cudagraph capture, where it would be taken from the graph's private
|
||||
|
||||
Reference in New Issue
Block a user