[Feature] Support batch invariance on ROCm (#52231)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Matt
2026-10-12 10:53:26 -05:00
committed by GitHub
co-authored by Claude Opus 5.5 Andreas Karatzas
parent 50b21a2667
commit e0317b28ce
31 changed files with 1106 additions and 122 deletions
@@ -6,6 +6,7 @@ from collections.abc import Callable
from typing import Any
import torch
import torch.distributed as dist
import vllm.envs as envs
from vllm.model_executor.determinism.batch_invariant_configs import (
@@ -59,6 +60,7 @@ def matmul_kernel_persistent(
stride_bn,
stride_cm,
stride_cn,
stride_bias,
BLOCK_SIZE_M: tl.constexpr, #
BLOCK_SIZE_N: tl.constexpr, #
BLOCK_SIZE_K: tl.constexpr, #
@@ -128,7 +130,7 @@ def matmul_kernel_persistent(
c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]
c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
if HAS_BIAS:
bias_ptrs = bias_ptr + offs_cn
bias_ptrs = bias_ptr + offs_cn * stride_bias
bias = tl.load(bias_ptrs, mask=offs_cn < N, other=0.0).to(tl.float32)
accumulator += bias
c = accumulator.to(c_ptr.dtype.element_ty)
@@ -305,6 +307,8 @@ def matmul_persistent(
),
)
if not _BLOCK_SIZES_RESOLVED:
_resolve_block_sizes()
configs = {
torch.bfloat16: {
"BLOCK_SIZE_M": 128,
@@ -345,6 +349,7 @@ def matmul_persistent(
b.stride(1), #
c.stride(0),
c.stride(1), #
bias.stride(0) if bias is not None else 1,
NUM_SMS=NUM_SMS, #
A_LARGE=a.numel() > 2**31,
B_LARGE=b.numel() > 2**31,
@@ -489,16 +494,19 @@ def bmm_kernel(
@triton.jit
def _log_softmax_kernel(
def _softmax_kernel(
input_ptr,
output_ptr,
input_row_stride,
input_col_stride,
output_row_stride,
n_cols,
BLOCK_SIZE: tl.constexpr,
LOG: tl.constexpr = False,
):
"""Compute log_softmax along the last dimension of a 2D tensor.
Each block handles one row of the input tensor.
"""Compute softmax, or log_softmax when ``LOG``, along the last dimension of a
2D tensor. Each block handles one row of the input tensor, in a fixed block
order, so the result never depends on the row count.
"""
# Get the row index for this block
row_idx = tl.program_id(0).to(tl.int64)
@@ -514,7 +522,9 @@ def _log_softmax_kernel(
mask = col_idx < n_cols
# Load values
vals = tl.load(row_start_ptr + col_idx, mask=mask, other=-float("inf"))
vals = tl.load(
row_start_ptr + col_idx * input_col_stride, mask=mask, other=-float("inf")
)
# Update maximum
max_val = tl.max(tl.maximum(vals, max_val))
@@ -526,25 +536,26 @@ def _log_softmax_kernel(
mask = col_idx < n_cols
# Load values
vals = tl.load(row_start_ptr + col_idx, mask=mask, other=0.0)
vals = tl.load(row_start_ptr + col_idx * input_col_stride, mask=mask, other=0.0)
# Compute exp(x - max_val) and accumulate
exp_vals = tl.exp(vals - max_val)
sum_exp += tl.sum(tl.where(mask, exp_vals, 0.0))
# Compute log(sum_exp)
log_sum_exp = tl.log(sum_exp)
log_sum_exp = tl.log(sum_exp) if LOG else 0.0
# Step 3: Compute final log_softmax values: x - max_val - log_sum_exp
# Step 3: normalise
for col_offset in range(0, n_cols, BLOCK_SIZE):
col_idx = col_offset + tl.arange(0, BLOCK_SIZE)
mask = col_idx < n_cols
# Load values
vals = tl.load(row_start_ptr + col_idx, mask=mask)
vals = tl.load(row_start_ptr + col_idx * input_col_stride, mask=mask)
# Compute log_softmax
output = vals - max_val - log_sum_exp
if LOG:
output = vals - max_val - log_sum_exp
else:
output = tl.exp(vals - max_val) / sum_exp
# Store results
tl.store(output_row_start_ptr + col_idx, output, mask=mask)
@@ -570,30 +581,55 @@ def log_softmax(input: torch.Tensor, dim: int = -1) -> torch.Tensor:
# Flatten all dimensions except the last one
original_shape = input.shape
input_2d = input.reshape(-1, input.shape[-1])
input_2d = input_2d.contiguous()
n_rows, n_cols = input_2d.shape
# Allocate output tensor
output = torch.empty_like(input_2d)
output = torch.empty((n_rows, n_cols), dtype=input_2d.dtype, device=input_2d.device)
# Choose block size based on the number of columns
BLOCK_SIZE = 1024
# Launch kernel with one block per row
grid = (n_rows,)
_log_softmax_kernel[grid](
_softmax_kernel[grid](
input_2d,
output,
input_2d.stride(0),
input_2d.stride(1),
output.stride(0),
n_cols,
BLOCK_SIZE=BLOCK_SIZE,
LOG=True,
)
# Reshape output back to original shape
return output.reshape(original_shape)
def softmax(input: torch.Tensor, dim: int = -1) -> torch.Tensor:
"""Compute softmax along the last dimension using a Triton kernel."""
if dim != -1 and dim != input.ndim - 1:
raise ValueError(
"This implementation only supports softmax along the last dimension"
)
original_shape = input.shape
input_2d = input.reshape(-1, input.shape[-1])
n_rows, n_cols = input_2d.shape
output = torch.empty((n_rows, n_cols), dtype=input_2d.dtype, device=input_2d.device)
_softmax_kernel[(n_rows,)](
input_2d,
output,
input_2d.stride(0),
input_2d.stride(1),
output.stride(0),
n_cols,
BLOCK_SIZE=1024,
)
return output.reshape(original_shape)
@triton.jit
def mean_kernel(
input_ptr,
@@ -735,6 +771,9 @@ def mean_dim(
def mm_batch_invariant(a, b, bias=None):
if a.device.type == "xpu":
# The descriptor kernel reads the bias at unit stride.
if bias is not None:
bias = bias.contiguous()
return matmul_descriptor_persistent(a, b, bias=bias)
return matmul_persistent(a, b, bias=bias)
@@ -820,6 +859,8 @@ def bmm_batch_invariant(a, b, *, out=None):
assert out.dtype == dtype and out.device == a.device, "out tensor mismatch"
c = out
if not _BLOCK_SIZES_RESOLVED:
_resolve_block_sizes()
configs = {
torch.bfloat16: {
"BLOCK_SIZE_M": 128,
@@ -877,8 +918,50 @@ def bmm_batch_invariant(a, b, *, out=None):
return c
def addmm_batch_invariant(bias, a, b):
return mm_batch_invariant(a, b, bias=bias)
def _addmm_fused_bias(bias, beta, alpha, n):
# The kernel adds bias into its fp32 accumulator indexed by output column,
# so it can only absorb a row vector of width n, unscaled by alpha.
if beta == 0 or alpha != 1:
return None
if bias.dim() != 1 or bias.shape[0] != n:
return None
# fp32 keeps the scaling exact; the kernel casts the bias to fp32 anyway.
return bias if beta == 1 else bias.float() * beta
def _addmm_impl(bias, a, b, beta, alpha, out):
fused_bias = _addmm_fused_bias(bias, beta, alpha, b.shape[1])
result = mm_batch_invariant(a, b, bias=fused_bias)
if fused_bias is None:
if alpha != 1:
result = result * alpha
if beta != 0:
result = result + (bias if beta == 1 else beta * bias)
if out is None:
return result
return out.copy_(result)
def addmm_batch_invariant(bias, a, b, *, beta=1, alpha=1):
return _addmm_impl(bias, a, b, beta, alpha, None)
# Inductor lowers mm/addmm/bmm to ``extern_kernels.<op>(..., out=buf)``, which
# dispatches the ``.out`` overload rather than the default one. Registering only
# the default leaves the compiled path on the vendor GEMM, so a compiled
# ``torch.addmm`` is batch variant while the eager one is not.
def mm_out_batch_invariant(a, b, *, out):
return matmul_batch_invariant(a, b, out=out)
def addmm_out_batch_invariant(bias, a, b, *, beta=1, alpha=1, out):
return _addmm_impl(bias, a, b, beta, alpha, out)
def bmm_out_batch_invariant(a, b, *, out):
return bmm_batch_invariant(a, b, out=out)
def _log_softmax_batch_invariant(input, dim, _half_to_float):
@@ -887,10 +970,16 @@ def _log_softmax_batch_invariant(input, dim, _half_to_float):
return log_softmax(input, dim=dim)
def softmax_batch_invariant(input, dim, dtype=None):
# Compute softmax in a deterministic way
# First subtract max for numerical stability (standard practice)
def softmax_batch_invariant(input, dim, half_to_float=False):
if half_to_float:
input = input.float()
if dim == -1 or dim == input.ndim - 1:
return softmax(input, dim=-1)
# Off the last dimension this falls back to torch ops, whose reduction split
# depends on the tensor shape.
input_max = torch.amax(input, dim=dim, keepdim=True)
# First subtract max for numerical stability (standard practice)
input = input - input_max
exp_x = torch.exp(input)
sum_exp_x = torch.sum(exp_x, dim=dim, keepdim=True)
@@ -1035,11 +1124,150 @@ def rms_norm_batch_invariant(
def linear_batch_invariant(input, weight, bias=None):
output = matmul_batch_invariant(input, weight.t())
if bias is not None:
output = output + bias
return output
bias = bias.expand(weight.shape[0])
output = mm_batch_invariant(
input.reshape(-1, input.shape[-1]), weight.t(), bias=bias
)
return output.view(*input.shape[:-1], output.shape[-1])
@triton.jit
def _fixed_order_sum_kernel(
src_ptr, # (WORLD_SIZE, numel) gathered contributions, rank-major
out_ptr, # (numel,)
numel,
stride_rank,
WORLD_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
LARGE: tl.constexpr,
):
"""Sum ``WORLD_SIZE`` contributions in ascending rank order, fp32 accumulator.
An output element sums exactly ``WORLD_SIZE`` values in a compile-time
constant order and rounds once, so the result depends only on the rank
ordering and never on how many elements are being reduced.
"""
pid = tl.program_id(0)
if LARGE:
pid = pid.to(tl.int64)
stride_rank = stride_rank.to(tl.int64)
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offs < numel
acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
for r in tl.static_range(WORLD_SIZE):
vals = tl.load(src_ptr + r * stride_rank + offs, mask=mask, other=0.0)
acc += vals.to(tl.float32)
tl.store(out_ptr + offs, acc.to(out_ptr.dtype.element_ty), mask=mask)
def all_reduce_batch_invariant(
input_: torch.Tensor, group: "torch.distributed.ProcessGroup | None" = None
) -> torch.Tensor:
"""Sum all-reduce whose result does not depend on the message size.
Library all-reduces (NCCL/RCCL) pick their algorithm, channel count and chunk
boundaries from the message size, so the order in which a given element's
contributions are summed changes with the number of tokens in the batch.
Instead, all-gather the contributions -- pure data movement, so bitwise
reproducible at any size -- and reduce them with ``_fixed_order_sum_kernel``.
"""
world_size = dist.get_world_size(group)
if world_size == 1:
return input_
x = input_.contiguous()
numel = x.numel()
gathered = torch.empty((world_size, numel), dtype=x.dtype, device=x.device)
dist.all_gather_into_tensor(gathered, x.view(-1), group=group)
out = torch.empty_like(x)
BLOCK_SIZE = 1024
grid = (triton.cdiv(numel, BLOCK_SIZE),)
_fixed_order_sum_kernel[grid](
gathered,
out.view(-1),
numel,
numel,
WORLD_SIZE=world_size,
BLOCK_SIZE=BLOCK_SIZE,
LARGE=gathered.numel() > 2**31,
)
return out
def reduce_scatter_batch_invariant(
input_: torch.Tensor,
group: "torch.distributed.ProcessGroup | None" = None,
sizes: list[int] | None = None,
) -> torch.Tensor:
"""Sum reduce-scatter over dim 0 whose result does not depend on the size.
``ncclReduceScatter`` picks its chunking from the message size just as
``ncclAllReduce`` does. Route around it the same way
``all_reduce_batch_invariant`` does: an all-to-all sends rank ``d`` exactly
the rows it owns from every rank -- pure data movement, so bitwise
reproducible however the library chunks it -- and lands them rank-major,
which is the layout ``_fixed_order_sum_kernel`` already consumes.
``sizes`` (the reduce-scatterv case) changes only *which* rows a rank
receives, never how the received contributions are summed, so variable
shard sizes are invariant for the same reason.
Args:
input_: Contiguous full-size input; dim 0 is the scattered axis.
group: Process group, defaults to the world group.
sizes: Per-rank row counts. Uniform split when ``None``.
Returns:
This rank's shard of the sum.
"""
world_size = dist.get_world_size(group)
if world_size == 1:
return input_
x = input_.contiguous()
if sizes is None:
assert x.shape[0] % world_size == 0
sizes = [x.shape[0] // world_size] * world_size
else:
assert len(sizes) == world_size
assert x.shape[0] == sum(sizes)
rows = sizes[dist.get_rank(group)]
out = torch.empty((rows, *x.shape[1:]), dtype=x.dtype, device=x.device)
recv = torch.empty(
(world_size * rows, *x.shape[1:]), dtype=x.dtype, device=x.device
)
# Every rank takes part even when its own shard is empty: it still has rows
# to send to the others.
dist.all_to_all_single(
recv,
x,
output_split_sizes=[rows] * world_size,
input_split_sizes=sizes,
group=group,
)
numel = out.numel()
if numel == 0:
return out
BLOCK_SIZE = 1024
grid = (triton.cdiv(numel, BLOCK_SIZE),)
_fixed_order_sum_kernel[grid](
recv.view(-1),
out.view(-1),
numel,
numel,
WORLD_SIZE=world_size,
BLOCK_SIZE=BLOCK_SIZE,
LARGE=recv.numel() > 2**31,
)
return out
_batch_invariant_MODE = False
@@ -1047,9 +1275,33 @@ _batch_invariant_LIB = None
_fp16_block_size_n = 256
_fp32_block_size_n = 128
_fp32_num_stages = 3
_BLOCK_SIZES_RESOLVED = False
_NUM_SMS: int = 0
def _resolve_block_sizes():
"""Size the tiles to the shared memory, to avoid triton OutOfResources.
Runs when the mode is enabled, or on first use by a caller that never
enables it, such as a kernel unit test.
"""
global _fp16_block_size_n, _fp32_block_size_n, _fp32_num_stages
global _BLOCK_SIZES_RESOLVED
_BLOCK_SIZES_RESOLVED = True
if not current_platform.is_cuda_alike():
return
if get_max_shared_memory_bytes() > 106496:
_fp16_block_size_n = 256
_fp32_block_size_n = 128
_fp32_num_stages = 3
else:
_fp16_block_size_n = 128
# SM89 N=1 fp32 128/stages=3 compiles to 131072B (>101376).
# Only N=1 OORs; keep 128/stages=3 for every other N.
_fp32_block_size_n = 32
_fp32_num_stages = 2
def enable_batch_invariant_mode():
global _batch_invariant_MODE, _batch_invariant_LIB
global _fp16_block_size_n, _fp32_block_size_n, _fp32_num_stages
@@ -1062,12 +1314,17 @@ def enable_batch_invariant_mode():
_batch_invariant_LIB = torch.library.Library("aten", "IMPL")
key = current_platform.dispatch_key
if current_platform.is_cuda():
if current_platform.is_device_capability_family(80):
# SM80 (Ampere) cannot rely on cuBLASLt-only determinism; install the
# triton persistent matmul overrides for mm/addmm/matmul/linear.
if current_platform.is_cuda_alike():
if current_platform.is_rocm() or current_platform.is_device_capability_family(
80
):
# Neither cuBLASLt on SM80 (Ampere) nor hipBLASLt can be made batch
# invariant; install the triton persistent matmul overrides for
# mm/addmm/matmul/linear.
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, key)
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, key)
_batch_invariant_LIB.impl("aten::mm.out", mm_out_batch_invariant, key)
_batch_invariant_LIB.impl("aten::addmm.out", addmm_out_batch_invariant, key)
_batch_invariant_LIB.impl("aten::matmul", matmul_batch_invariant, key)
_batch_invariant_LIB.impl("aten::linear", linear_batch_invariant, key)
else:
@@ -1077,18 +1334,7 @@ def enable_batch_invariant_mode():
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":16:8"
os.environ["CUBLASLT_WORKSPACE_SIZE"] = "1"
# Query the shared memory size and set block size
# accordingly to avoid triton OutOfResources.
if get_max_shared_memory_bytes() > 106496:
_fp16_block_size_n = 256
_fp32_block_size_n = 128
_fp32_num_stages = 3
else:
_fp16_block_size_n = 128
# SM89 N=1 fp32 128/stages=3 compiles to 131072B (>101376).
# Only N=1 OORs; keep 128/stages=3 for every other N.
_fp32_block_size_n = 32
_fp32_num_stages = 2
_resolve_block_sizes()
elif current_platform.is_xpu():
# Tensor descriptors need a global-memory allocator; must be set outside
# torch.compile regions (triton.set_allocator modifies global state).
@@ -1100,6 +1346,8 @@ def enable_batch_invariant_mode():
_NUM_SMS = num_compute_units(0)
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, key)
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, key)
_batch_invariant_LIB.impl("aten::mm.out", mm_out_batch_invariant, key)
_batch_invariant_LIB.impl("aten::addmm.out", addmm_out_batch_invariant, key)
_batch_invariant_LIB.impl("aten::matmul", matmul_batch_invariant, key)
_batch_invariant_LIB.impl("aten::linear", linear_batch_invariant, key)
@@ -1123,7 +1371,9 @@ def enable_batch_invariant_mode():
_batch_invariant_LIB.impl(
"aten::bmm", bmm_batch_invariant, key, allow_override=True
)
torch.bmm = bmm_batch_invariant
_batch_invariant_LIB.impl(
"aten::bmm.out", bmm_out_batch_invariant, key, allow_override=True
)
reduced_precision_val = (
(False, False) if is_torch_equal_or_newer("2.10.0") else False
@@ -1145,28 +1395,32 @@ def override_envs_for_invariance():
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
# NCCL determinism settings. NCCL keeps the launch mode process-wide, so,
# unlike the pins below, it also applies to communicators shared with other
# processes.
os.environ["NCCL_LAUNCH_MODE"] = "GROUP"
pin_nccl_env(
{
"NCCL_COLLNET_ENABLE": "0",
"NCCL_NVLS_ENABLE": "0",
"NCCL_P2P_NET_DISABLE": "1",
"NCCL_MIN_NCHANNELS": "1",
"NCCL_MAX_NCHANNELS": "1",
"NCCL_PROTO": "Simple",
# NCCL >= 2.31 zero-fills the algorithm table of every collective
# when NCCL_ALGO is set and re-enables only the named ones; together
# with the NCCL_PROTO above, collectives not named here end up with
# no algorithm and fail with ncclInvalidUsage. Re-enable Ring and
# Tree for all collectives, then pin AllReduce to Tree.
"NCCL_ALGO": "ring,tree;allreduce:tree",
"NCCL_NTHREADS": "1",
"NCCL_SOCKET_NTHREADS": "1",
}
)
if current_platform.is_rocm():
# The skinny GEMMs pick their split of K from the batch size.
os.environ["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
else:
# NCCL determinism settings. NCCL keeps the launch mode process-wide, so,
# unlike the pins below, it also applies to communicators shared with other
# processes.
os.environ["NCCL_LAUNCH_MODE"] = "GROUP"
pin_nccl_env(
{
"NCCL_COLLNET_ENABLE": "0",
"NCCL_NVLS_ENABLE": "0",
"NCCL_P2P_NET_DISABLE": "1",
"NCCL_MIN_NCHANNELS": "1",
"NCCL_MAX_NCHANNELS": "1",
"NCCL_PROTO": "Simple",
# NCCL >= 2.31 zero-fills the algorithm table of every collective
# when NCCL_ALGO is set and re-enables only the named ones; together
# with the NCCL_PROTO above, collectives not named here end up with
# no algorithm and fail with ncclInvalidUsage. Re-enable Ring and
# Tree for all collectives, then pin AllReduce to Tree.
"NCCL_ALGO": "ring,tree;allreduce:tree",
"NCCL_NTHREADS": "1",
"NCCL_SOCKET_NTHREADS": "1",
}
)
# torch.compile settings
os.environ["VLLM_USE_AOT_COMPILE"] = "0"
@@ -218,6 +218,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.rocm import (
)
from vllm.model_executor.kernels.linear.scaled_mm.triton import (
TritonFp8BlockScaledMMKernel,
TritonFP8ScaledMMLinearKernel,
TritonInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
@@ -253,6 +254,20 @@ def _get_linear_backend(*, quantization: str) -> str:
return config.kernel_config.linear_backend
def _batch_invariant_fp8_kernel(kernel, config, quantization: str):
can_implement, reason = is_supported_and_can_implement_kernel(kernel, config, None)
if not can_implement:
raise ValueError(f"Batch invariance on ROCm: {reason}")
linear_backend = _get_linear_backend(quantization=quantization)
if linear_backend not in ("auto", "triton"):
logger.warning_once(
"VLLM_BATCH_INVARIANT overrides --linear-backend=%s; "
"using the Triton backend for deterministic execution.",
linear_backend,
)
return kernel
# Kernel classes covered by each --linear-backend value.
_LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
"b12x": {
@@ -723,6 +738,13 @@ def init_fp8_linear_kernel(
)
if activation_quant_key.scale.group_shape.is_per_group():
if envs.VLLM_BATCH_INVARIANT and current_platform.is_rocm():
force_kernel = _batch_invariant_fp8_kernel(
TritonFp8BlockScaledMMKernel,
scaled_mm_linear_kernel_config,
"fp8_block_w8a8",
)
kernel_type = choose_scaled_mm_linear_kernel(
config=scaled_mm_linear_kernel_config,
possible_kernels=_POSSIBLE_FP8_BLOCK_KERNELS, # type: ignore[misc]
@@ -755,6 +777,13 @@ def init_fp8_linear_kernel(
)
else:
if envs.VLLM_BATCH_INVARIANT and current_platform.is_rocm():
force_kernel = _batch_invariant_fp8_kernel(
TritonFP8ScaledMMLinearKernel,
scaled_mm_linear_kernel_config,
"fp8_w8a8",
)
kernel_type = choose_scaled_mm_linear_kernel(
config=scaled_mm_linear_kernel_config,
possible_kernels=_POSSIBLE_FP8_KERNELS, # type: ignore[arg-type]
@@ -780,6 +809,19 @@ def init_fp8_linear_kernel(
)
def fp8_scaled_mm_is_batch_invariant(
kernel: FP8ScaledMMLinearKernel | Fp8BlockScaledMMLinearKernel,
) -> bool:
"""Whether a row's output from the kernel is independent of the other rows."""
if not isinstance(
kernel, (CutlassFP8ScaledMMLinearKernel, TritonFP8ScaledMMLinearKernel)
):
return False
# A dynamic per-tensor activation scale is an amax over every row.
scale = kernel.config.activation_quant_key.scale
return scale.static or not scale.group_shape.is_per_tensor()
def init_int8_linear_kernel(
is_channelwise: bool,
is_static_input_scheme: bool,
@@ -1250,6 +1292,7 @@ def register_linear_kernel(
__all__ = [
"fp8_scaled_mm_is_batch_invariant",
"init_fp8_linear_kernel",
"init_int8_linear_kernel",
"init_nvfp4_linear_kernel",
@@ -62,7 +62,12 @@ if is_aiter_found_and_supported():
N = weight.shape[0]
K = weight.shape[1]
if rocm_use_aiter_fp4_asm_gemm:
if M <= 64 and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K):
# The preshuffle kernel's tuned configs split K in some M buckets.
if (
M <= 64
and not envs.VLLM_BATCH_INVARIANT
and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K)
):
if x_scales is None:
# use hip quant kernel for performance
if M >= 32:
@@ -116,7 +121,16 @@ if is_aiter_found_and_supported():
x_q.shape[0], weight.shape[0], device=x_q.device, dtype=out_dtype
)
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y)
config = None
if envs.VLLM_BATCH_INVARIANT:
# The configs split K in some M buckets (up to M <= 256 for
# tuned shapes). Keep the tile, drop the split.
from aiter.ops.triton.gemm_afp4wfp4 import _get_config
config, _ = _get_config(M, N, K)
config = dict(config, NUM_KSPLIT=1)
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y, config)
return y
def gemm_with_dynamic_quant_fake(
@@ -12,6 +12,7 @@ instead, since ``can_implement`` does not filter on K.
import torch
from torch.nn.parameter import Parameter
import vllm.envs as envs
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
@@ -246,7 +247,10 @@ class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
N, K = weight.shape
scale_k = K // MXFP8_BLOCK_SIZE
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
block_scale = _as_block32_scale(weight_scale)
# rocm_mxfp8_block32_gemm's split-K varies with M.
block_scale = (
None if envs.VLLM_BATCH_INVARIANT else _as_block32_scale(weight_scale)
)
if block_scale is not None:
# Checkpoints with 32x32 scale blocks (DeepSeek V4/V4.1) get them
# back from the loader expanded per row; keep the blocks and use
@@ -39,6 +39,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.triton import (
TritonFP8ScaledMMLinearKernel,
TritonInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
@@ -65,6 +66,7 @@ __all__ = [
"PerTensorTorchFP8ScaledMMLinearKernel",
"RowWiseTorchFP8ScaledMMLinearKernel",
"ROCmFP8ScaledMMLinearKernel",
"TritonFP8ScaledMMLinearKernel",
"TritonInt8ScaledMMLinearKernel",
"ZentorchInt8ScaledMMLinearKernel",
"Fp8BlockScaledMMLinearKernel",
@@ -20,6 +20,8 @@ from .BlockScaledMMLinearKernel import (
)
from .cutlass import CutlassInt8ScaledMMLinearKernel
from .ScaledMMLinearKernel import (
FP8ScaledMMLinearKernel,
FP8ScaledMMLinearLayerConfig,
Int8ScaledMMLinearLayerConfig,
)
@@ -156,6 +158,46 @@ class TritonInt8ScaledMMLinearKernel(CutlassInt8ScaledMMLinearKernel):
return out
class TritonFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cuda_alike():
return False, "requires ROCm or CUDA."
return True, None
@classmethod
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
act_group = c.activation_quant_key.scale.group_shape
weight_group = c.weight_quant_key.scale.group_shape
if not (act_group.is_per_tensor() or act_group.is_per_token()):
return False, "requires per-tensor or per-token activation scales."
if not (
weight_group.is_per_tensor()
or weight_group.is_per_channel()
or weight_group.is_per_token()
):
return False, "requires per-tensor, per-channel or per-token weight scales."
return True, None
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
out = triton_scaled_mm(
A, B, scale_a=As, scale_b=Bs, out_dtype=out_dtype, bias=bias
)
return out.view(*output_shape)
class TritonFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
@classmethod
def is_supported(cls, compute_capability=None):
@@ -51,6 +51,10 @@ class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
def _supports_parallel_config(moe_parallel_config) -> bool:
return True
@staticmethod
def _supports_batch_invariance() -> bool:
return False
@staticmethod
def is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
@@ -218,7 +218,7 @@ class OCP_MXQuantizationEmulationTritonExperts(TritonExperts):
global_num_experts=global_num_experts,
expert_map=expert_map,
a1q_scale=None,
a2_scale=None,
a2_scale=a2_scale,
workspace13=workspace13,
workspace2=workspace2,
expert_tokens_meta=expert_tokens_meta,
@@ -150,16 +150,16 @@ def scaled_mm_kernel(
scale_b = scale_b.broadcast_to((BLOCK_SIZE_N, 1))
accumulator = scale_b.T * accumulator.to(tl.float32)
# Convert to output format.
c = accumulator.to(c_ptr.type.element_ty)
# Add bias, it's already in output format, so add it after conversion.
# Add bias in accumulator dtype for less rounding error.
if bias_ptr:
offsets_bias = offsets_bn
bias_ptrs = bias_ptr + offsets_bias
bias_mask = offsets_bias < N
bias = tl.load(bias_ptrs, bias_mask)
c += bias
accumulator += bias
# Convert to output format.
c = accumulator.to(c_ptr.type.element_ty)
# Save output
offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64)
@@ -224,6 +224,8 @@ def triton_scaled_mm(
tile_shape = (64, 128, 128)
else:
tile_shape = (128, 128, 128)
else:
tile_shape = (block_size_m, block_size_n, block_size_k)
block_size_m, block_size_n, block_size_k = tile_shape
@@ -9,10 +9,10 @@ import vllm.envs as envs
from vllm.config import get_current_vllm_config
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.model_executor.kernels.linear import (
fp8_scaled_mm_is_batch_invariant,
init_fp8_linear_kernel,
)
from vllm.model_executor.kernels.linear.scaled_mm import (
CutlassFP8ScaledMMLinearKernel,
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.attention import Attention
@@ -432,7 +432,7 @@ class Fp8LinearMethod(LinearMethodBase):
bias,
)
else:
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
if fp8_scaled_mm_is_batch_invariant(self.fp8_linear):
return self.fp8_linear.apply_weights(layer, x, bias)
# per-tensor/channel: dequant to BF16 and run GEMM
@@ -15,9 +15,11 @@ if TYPE_CHECKING:
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.config import get_current_vllm_config
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
from vllm.model_executor.kernels.linear import (
fp8_scaled_mm_is_batch_invariant,
init_fp8_linear_kernel,
)
from vllm.model_executor.kernels.linear.scaled_mm import (
CutlassFP8ScaledMMLinearKernel,
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.fused_moe import RoutedExperts
@@ -278,7 +280,7 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
) -> torch.Tensor:
# if batch invariant mode is enabled, use BF16 dequant
if envs.VLLM_BATCH_INVARIANT:
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
if fp8_scaled_mm_is_batch_invariant(self.fp8_linear):
return self.fp8_linear.apply_weights(layer, x, bias)
weight_fp8 = layer.weight.to(torch.bfloat16)
@@ -466,7 +468,7 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
# if batch invariant mode is enabled dequant
if (
envs.VLLM_BATCH_INVARIANT
and not isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel)
and not fp8_scaled_mm_is_batch_invariant(self.fp8_linear)
and not isinstance(x, QuantizedActivation)
):
weight_dequant = (
@@ -1041,7 +1041,10 @@ def w8a8_triton_block_scaled_mm(
C_shape = A.shape[:-1] + (N,)
C = A.new_empty(C_shape, dtype=output_dtype)
configs = get_w8a8_block_fp8_configs(N, K, block_size[0], block_size[1])
if envs.VLLM_BATCH_INVARIANT:
configs = None # use the default config
else:
configs = get_w8a8_block_fp8_configs(N, K, block_size[0], block_size[1])
if configs:
# Get the optimal config if there is one
config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
+7
View File
@@ -288,6 +288,13 @@ def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, boo
def rocm_unquantized_gemm_impl(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
if envs.VLLM_BATCH_INVARIANT:
from vllm.model_executor.determinism.batch_invariant import (
linear_batch_invariant,
)
return linear_batch_invariant(x, weight, bias)
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx950, on_gfx1250
n = x.numel() // x.size(-1)