[Feature] Support batch invariance on ROCm (#52231)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Andreas Karatzas
parent
50b21a2667
commit
e0317b28ce
@@ -6,6 +6,7 @@ from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.determinism.batch_invariant_configs import (
|
||||
@@ -59,6 +60,7 @@ def matmul_kernel_persistent(
|
||||
stride_bn,
|
||||
stride_cm,
|
||||
stride_cn,
|
||||
stride_bias,
|
||||
BLOCK_SIZE_M: tl.constexpr, #
|
||||
BLOCK_SIZE_N: tl.constexpr, #
|
||||
BLOCK_SIZE_K: tl.constexpr, #
|
||||
@@ -128,7 +130,7 @@ def matmul_kernel_persistent(
|
||||
c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]
|
||||
c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
|
||||
if HAS_BIAS:
|
||||
bias_ptrs = bias_ptr + offs_cn
|
||||
bias_ptrs = bias_ptr + offs_cn * stride_bias
|
||||
bias = tl.load(bias_ptrs, mask=offs_cn < N, other=0.0).to(tl.float32)
|
||||
accumulator += bias
|
||||
c = accumulator.to(c_ptr.dtype.element_ty)
|
||||
@@ -305,6 +307,8 @@ def matmul_persistent(
|
||||
),
|
||||
)
|
||||
|
||||
if not _BLOCK_SIZES_RESOLVED:
|
||||
_resolve_block_sizes()
|
||||
configs = {
|
||||
torch.bfloat16: {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
@@ -345,6 +349,7 @@ def matmul_persistent(
|
||||
b.stride(1), #
|
||||
c.stride(0),
|
||||
c.stride(1), #
|
||||
bias.stride(0) if bias is not None else 1,
|
||||
NUM_SMS=NUM_SMS, #
|
||||
A_LARGE=a.numel() > 2**31,
|
||||
B_LARGE=b.numel() > 2**31,
|
||||
@@ -489,16 +494,19 @@ def bmm_kernel(
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _log_softmax_kernel(
|
||||
def _softmax_kernel(
|
||||
input_ptr,
|
||||
output_ptr,
|
||||
input_row_stride,
|
||||
input_col_stride,
|
||||
output_row_stride,
|
||||
n_cols,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
LOG: tl.constexpr = False,
|
||||
):
|
||||
"""Compute log_softmax along the last dimension of a 2D tensor.
|
||||
Each block handles one row of the input tensor.
|
||||
"""Compute softmax, or log_softmax when ``LOG``, along the last dimension of a
|
||||
2D tensor. Each block handles one row of the input tensor, in a fixed block
|
||||
order, so the result never depends on the row count.
|
||||
"""
|
||||
# Get the row index for this block
|
||||
row_idx = tl.program_id(0).to(tl.int64)
|
||||
@@ -514,7 +522,9 @@ def _log_softmax_kernel(
|
||||
mask = col_idx < n_cols
|
||||
|
||||
# Load values
|
||||
vals = tl.load(row_start_ptr + col_idx, mask=mask, other=-float("inf"))
|
||||
vals = tl.load(
|
||||
row_start_ptr + col_idx * input_col_stride, mask=mask, other=-float("inf")
|
||||
)
|
||||
|
||||
# Update maximum
|
||||
max_val = tl.max(tl.maximum(vals, max_val))
|
||||
@@ -526,25 +536,26 @@ def _log_softmax_kernel(
|
||||
mask = col_idx < n_cols
|
||||
|
||||
# Load values
|
||||
vals = tl.load(row_start_ptr + col_idx, mask=mask, other=0.0)
|
||||
vals = tl.load(row_start_ptr + col_idx * input_col_stride, mask=mask, other=0.0)
|
||||
|
||||
# Compute exp(x - max_val) and accumulate
|
||||
exp_vals = tl.exp(vals - max_val)
|
||||
sum_exp += tl.sum(tl.where(mask, exp_vals, 0.0))
|
||||
|
||||
# Compute log(sum_exp)
|
||||
log_sum_exp = tl.log(sum_exp)
|
||||
log_sum_exp = tl.log(sum_exp) if LOG else 0.0
|
||||
|
||||
# Step 3: Compute final log_softmax values: x - max_val - log_sum_exp
|
||||
# Step 3: normalise
|
||||
for col_offset in range(0, n_cols, BLOCK_SIZE):
|
||||
col_idx = col_offset + tl.arange(0, BLOCK_SIZE)
|
||||
mask = col_idx < n_cols
|
||||
|
||||
# Load values
|
||||
vals = tl.load(row_start_ptr + col_idx, mask=mask)
|
||||
vals = tl.load(row_start_ptr + col_idx * input_col_stride, mask=mask)
|
||||
|
||||
# Compute log_softmax
|
||||
output = vals - max_val - log_sum_exp
|
||||
if LOG:
|
||||
output = vals - max_val - log_sum_exp
|
||||
else:
|
||||
output = tl.exp(vals - max_val) / sum_exp
|
||||
|
||||
# Store results
|
||||
tl.store(output_row_start_ptr + col_idx, output, mask=mask)
|
||||
@@ -570,30 +581,55 @@ def log_softmax(input: torch.Tensor, dim: int = -1) -> torch.Tensor:
|
||||
# Flatten all dimensions except the last one
|
||||
original_shape = input.shape
|
||||
input_2d = input.reshape(-1, input.shape[-1])
|
||||
input_2d = input_2d.contiguous()
|
||||
|
||||
n_rows, n_cols = input_2d.shape
|
||||
|
||||
# Allocate output tensor
|
||||
output = torch.empty_like(input_2d)
|
||||
output = torch.empty((n_rows, n_cols), dtype=input_2d.dtype, device=input_2d.device)
|
||||
|
||||
# Choose block size based on the number of columns
|
||||
BLOCK_SIZE = 1024
|
||||
|
||||
# Launch kernel with one block per row
|
||||
grid = (n_rows,)
|
||||
_log_softmax_kernel[grid](
|
||||
_softmax_kernel[grid](
|
||||
input_2d,
|
||||
output,
|
||||
input_2d.stride(0),
|
||||
input_2d.stride(1),
|
||||
output.stride(0),
|
||||
n_cols,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
LOG=True,
|
||||
)
|
||||
# Reshape output back to original shape
|
||||
return output.reshape(original_shape)
|
||||
|
||||
|
||||
def softmax(input: torch.Tensor, dim: int = -1) -> torch.Tensor:
|
||||
"""Compute softmax along the last dimension using a Triton kernel."""
|
||||
if dim != -1 and dim != input.ndim - 1:
|
||||
raise ValueError(
|
||||
"This implementation only supports softmax along the last dimension"
|
||||
)
|
||||
|
||||
original_shape = input.shape
|
||||
input_2d = input.reshape(-1, input.shape[-1])
|
||||
n_rows, n_cols = input_2d.shape
|
||||
output = torch.empty((n_rows, n_cols), dtype=input_2d.dtype, device=input_2d.device)
|
||||
|
||||
_softmax_kernel[(n_rows,)](
|
||||
input_2d,
|
||||
output,
|
||||
input_2d.stride(0),
|
||||
input_2d.stride(1),
|
||||
output.stride(0),
|
||||
n_cols,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
return output.reshape(original_shape)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def mean_kernel(
|
||||
input_ptr,
|
||||
@@ -735,6 +771,9 @@ def mean_dim(
|
||||
|
||||
def mm_batch_invariant(a, b, bias=None):
|
||||
if a.device.type == "xpu":
|
||||
# The descriptor kernel reads the bias at unit stride.
|
||||
if bias is not None:
|
||||
bias = bias.contiguous()
|
||||
return matmul_descriptor_persistent(a, b, bias=bias)
|
||||
return matmul_persistent(a, b, bias=bias)
|
||||
|
||||
@@ -820,6 +859,8 @@ def bmm_batch_invariant(a, b, *, out=None):
|
||||
assert out.dtype == dtype and out.device == a.device, "out tensor mismatch"
|
||||
c = out
|
||||
|
||||
if not _BLOCK_SIZES_RESOLVED:
|
||||
_resolve_block_sizes()
|
||||
configs = {
|
||||
torch.bfloat16: {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
@@ -877,8 +918,50 @@ def bmm_batch_invariant(a, b, *, out=None):
|
||||
return c
|
||||
|
||||
|
||||
def addmm_batch_invariant(bias, a, b):
|
||||
return mm_batch_invariant(a, b, bias=bias)
|
||||
def _addmm_fused_bias(bias, beta, alpha, n):
|
||||
# The kernel adds bias into its fp32 accumulator indexed by output column,
|
||||
# so it can only absorb a row vector of width n, unscaled by alpha.
|
||||
if beta == 0 or alpha != 1:
|
||||
return None
|
||||
if bias.dim() != 1 or bias.shape[0] != n:
|
||||
return None
|
||||
# fp32 keeps the scaling exact; the kernel casts the bias to fp32 anyway.
|
||||
return bias if beta == 1 else bias.float() * beta
|
||||
|
||||
|
||||
def _addmm_impl(bias, a, b, beta, alpha, out):
|
||||
fused_bias = _addmm_fused_bias(bias, beta, alpha, b.shape[1])
|
||||
result = mm_batch_invariant(a, b, bias=fused_bias)
|
||||
if fused_bias is None:
|
||||
if alpha != 1:
|
||||
result = result * alpha
|
||||
if beta != 0:
|
||||
result = result + (bias if beta == 1 else beta * bias)
|
||||
if out is None:
|
||||
return result
|
||||
return out.copy_(result)
|
||||
|
||||
|
||||
def addmm_batch_invariant(bias, a, b, *, beta=1, alpha=1):
|
||||
return _addmm_impl(bias, a, b, beta, alpha, None)
|
||||
|
||||
|
||||
# Inductor lowers mm/addmm/bmm to ``extern_kernels.<op>(..., out=buf)``, which
|
||||
# dispatches the ``.out`` overload rather than the default one. Registering only
|
||||
# the default leaves the compiled path on the vendor GEMM, so a compiled
|
||||
# ``torch.addmm`` is batch variant while the eager one is not.
|
||||
|
||||
|
||||
def mm_out_batch_invariant(a, b, *, out):
|
||||
return matmul_batch_invariant(a, b, out=out)
|
||||
|
||||
|
||||
def addmm_out_batch_invariant(bias, a, b, *, beta=1, alpha=1, out):
|
||||
return _addmm_impl(bias, a, b, beta, alpha, out)
|
||||
|
||||
|
||||
def bmm_out_batch_invariant(a, b, *, out):
|
||||
return bmm_batch_invariant(a, b, out=out)
|
||||
|
||||
|
||||
def _log_softmax_batch_invariant(input, dim, _half_to_float):
|
||||
@@ -887,10 +970,16 @@ def _log_softmax_batch_invariant(input, dim, _half_to_float):
|
||||
return log_softmax(input, dim=dim)
|
||||
|
||||
|
||||
def softmax_batch_invariant(input, dim, dtype=None):
|
||||
# Compute softmax in a deterministic way
|
||||
# First subtract max for numerical stability (standard practice)
|
||||
def softmax_batch_invariant(input, dim, half_to_float=False):
|
||||
if half_to_float:
|
||||
input = input.float()
|
||||
if dim == -1 or dim == input.ndim - 1:
|
||||
return softmax(input, dim=-1)
|
||||
|
||||
# Off the last dimension this falls back to torch ops, whose reduction split
|
||||
# depends on the tensor shape.
|
||||
input_max = torch.amax(input, dim=dim, keepdim=True)
|
||||
# First subtract max for numerical stability (standard practice)
|
||||
input = input - input_max
|
||||
exp_x = torch.exp(input)
|
||||
sum_exp_x = torch.sum(exp_x, dim=dim, keepdim=True)
|
||||
@@ -1035,11 +1124,150 @@ def rms_norm_batch_invariant(
|
||||
|
||||
|
||||
def linear_batch_invariant(input, weight, bias=None):
|
||||
output = matmul_batch_invariant(input, weight.t())
|
||||
|
||||
if bias is not None:
|
||||
output = output + bias
|
||||
return output
|
||||
bias = bias.expand(weight.shape[0])
|
||||
output = mm_batch_invariant(
|
||||
input.reshape(-1, input.shape[-1]), weight.t(), bias=bias
|
||||
)
|
||||
return output.view(*input.shape[:-1], output.shape[-1])
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fixed_order_sum_kernel(
|
||||
src_ptr, # (WORLD_SIZE, numel) gathered contributions, rank-major
|
||||
out_ptr, # (numel,)
|
||||
numel,
|
||||
stride_rank,
|
||||
WORLD_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
LARGE: tl.constexpr,
|
||||
):
|
||||
"""Sum ``WORLD_SIZE`` contributions in ascending rank order, fp32 accumulator.
|
||||
|
||||
An output element sums exactly ``WORLD_SIZE`` values in a compile-time
|
||||
constant order and rounds once, so the result depends only on the rank
|
||||
ordering and never on how many elements are being reduced.
|
||||
"""
|
||||
pid = tl.program_id(0)
|
||||
if LARGE:
|
||||
pid = pid.to(tl.int64)
|
||||
stride_rank = stride_rank.to(tl.int64)
|
||||
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offs < numel
|
||||
|
||||
acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
|
||||
for r in tl.static_range(WORLD_SIZE):
|
||||
vals = tl.load(src_ptr + r * stride_rank + offs, mask=mask, other=0.0)
|
||||
acc += vals.to(tl.float32)
|
||||
|
||||
tl.store(out_ptr + offs, acc.to(out_ptr.dtype.element_ty), mask=mask)
|
||||
|
||||
|
||||
def all_reduce_batch_invariant(
|
||||
input_: torch.Tensor, group: "torch.distributed.ProcessGroup | None" = None
|
||||
) -> torch.Tensor:
|
||||
"""Sum all-reduce whose result does not depend on the message size.
|
||||
|
||||
Library all-reduces (NCCL/RCCL) pick their algorithm, channel count and chunk
|
||||
boundaries from the message size, so the order in which a given element's
|
||||
contributions are summed changes with the number of tokens in the batch.
|
||||
|
||||
Instead, all-gather the contributions -- pure data movement, so bitwise
|
||||
reproducible at any size -- and reduce them with ``_fixed_order_sum_kernel``.
|
||||
"""
|
||||
world_size = dist.get_world_size(group)
|
||||
if world_size == 1:
|
||||
return input_
|
||||
|
||||
x = input_.contiguous()
|
||||
numel = x.numel()
|
||||
gathered = torch.empty((world_size, numel), dtype=x.dtype, device=x.device)
|
||||
dist.all_gather_into_tensor(gathered, x.view(-1), group=group)
|
||||
|
||||
out = torch.empty_like(x)
|
||||
BLOCK_SIZE = 1024
|
||||
grid = (triton.cdiv(numel, BLOCK_SIZE),)
|
||||
_fixed_order_sum_kernel[grid](
|
||||
gathered,
|
||||
out.view(-1),
|
||||
numel,
|
||||
numel,
|
||||
WORLD_SIZE=world_size,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
LARGE=gathered.numel() > 2**31,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def reduce_scatter_batch_invariant(
|
||||
input_: torch.Tensor,
|
||||
group: "torch.distributed.ProcessGroup | None" = None,
|
||||
sizes: list[int] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Sum reduce-scatter over dim 0 whose result does not depend on the size.
|
||||
|
||||
``ncclReduceScatter`` picks its chunking from the message size just as
|
||||
``ncclAllReduce`` does. Route around it the same way
|
||||
``all_reduce_batch_invariant`` does: an all-to-all sends rank ``d`` exactly
|
||||
the rows it owns from every rank -- pure data movement, so bitwise
|
||||
reproducible however the library chunks it -- and lands them rank-major,
|
||||
which is the layout ``_fixed_order_sum_kernel`` already consumes.
|
||||
|
||||
``sizes`` (the reduce-scatterv case) changes only *which* rows a rank
|
||||
receives, never how the received contributions are summed, so variable
|
||||
shard sizes are invariant for the same reason.
|
||||
|
||||
Args:
|
||||
input_: Contiguous full-size input; dim 0 is the scattered axis.
|
||||
group: Process group, defaults to the world group.
|
||||
sizes: Per-rank row counts. Uniform split when ``None``.
|
||||
|
||||
Returns:
|
||||
This rank's shard of the sum.
|
||||
|
||||
"""
|
||||
world_size = dist.get_world_size(group)
|
||||
if world_size == 1:
|
||||
return input_
|
||||
|
||||
x = input_.contiguous()
|
||||
if sizes is None:
|
||||
assert x.shape[0] % world_size == 0
|
||||
sizes = [x.shape[0] // world_size] * world_size
|
||||
else:
|
||||
assert len(sizes) == world_size
|
||||
assert x.shape[0] == sum(sizes)
|
||||
|
||||
rows = sizes[dist.get_rank(group)]
|
||||
out = torch.empty((rows, *x.shape[1:]), dtype=x.dtype, device=x.device)
|
||||
recv = torch.empty(
|
||||
(world_size * rows, *x.shape[1:]), dtype=x.dtype, device=x.device
|
||||
)
|
||||
# Every rank takes part even when its own shard is empty: it still has rows
|
||||
# to send to the others.
|
||||
dist.all_to_all_single(
|
||||
recv,
|
||||
x,
|
||||
output_split_sizes=[rows] * world_size,
|
||||
input_split_sizes=sizes,
|
||||
group=group,
|
||||
)
|
||||
|
||||
numel = out.numel()
|
||||
if numel == 0:
|
||||
return out
|
||||
BLOCK_SIZE = 1024
|
||||
grid = (triton.cdiv(numel, BLOCK_SIZE),)
|
||||
_fixed_order_sum_kernel[grid](
|
||||
recv.view(-1),
|
||||
out.view(-1),
|
||||
numel,
|
||||
numel,
|
||||
WORLD_SIZE=world_size,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
LARGE=recv.numel() > 2**31,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
_batch_invariant_MODE = False
|
||||
@@ -1047,9 +1275,33 @@ _batch_invariant_LIB = None
|
||||
_fp16_block_size_n = 256
|
||||
_fp32_block_size_n = 128
|
||||
_fp32_num_stages = 3
|
||||
_BLOCK_SIZES_RESOLVED = False
|
||||
_NUM_SMS: int = 0
|
||||
|
||||
|
||||
def _resolve_block_sizes():
|
||||
"""Size the tiles to the shared memory, to avoid triton OutOfResources.
|
||||
|
||||
Runs when the mode is enabled, or on first use by a caller that never
|
||||
enables it, such as a kernel unit test.
|
||||
"""
|
||||
global _fp16_block_size_n, _fp32_block_size_n, _fp32_num_stages
|
||||
global _BLOCK_SIZES_RESOLVED
|
||||
_BLOCK_SIZES_RESOLVED = True
|
||||
if not current_platform.is_cuda_alike():
|
||||
return
|
||||
if get_max_shared_memory_bytes() > 106496:
|
||||
_fp16_block_size_n = 256
|
||||
_fp32_block_size_n = 128
|
||||
_fp32_num_stages = 3
|
||||
else:
|
||||
_fp16_block_size_n = 128
|
||||
# SM89 N=1 fp32 128/stages=3 compiles to 131072B (>101376).
|
||||
# Only N=1 OORs; keep 128/stages=3 for every other N.
|
||||
_fp32_block_size_n = 32
|
||||
_fp32_num_stages = 2
|
||||
|
||||
|
||||
def enable_batch_invariant_mode():
|
||||
global _batch_invariant_MODE, _batch_invariant_LIB
|
||||
global _fp16_block_size_n, _fp32_block_size_n, _fp32_num_stages
|
||||
@@ -1062,12 +1314,17 @@ def enable_batch_invariant_mode():
|
||||
_batch_invariant_LIB = torch.library.Library("aten", "IMPL")
|
||||
|
||||
key = current_platform.dispatch_key
|
||||
if current_platform.is_cuda():
|
||||
if current_platform.is_device_capability_family(80):
|
||||
# SM80 (Ampere) cannot rely on cuBLASLt-only determinism; install the
|
||||
# triton persistent matmul overrides for mm/addmm/matmul/linear.
|
||||
if current_platform.is_cuda_alike():
|
||||
if current_platform.is_rocm() or current_platform.is_device_capability_family(
|
||||
80
|
||||
):
|
||||
# Neither cuBLASLt on SM80 (Ampere) nor hipBLASLt can be made batch
|
||||
# invariant; install the triton persistent matmul overrides for
|
||||
# mm/addmm/matmul/linear.
|
||||
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::mm.out", mm_out_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::addmm.out", addmm_out_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::matmul", matmul_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::linear", linear_batch_invariant, key)
|
||||
else:
|
||||
@@ -1077,18 +1334,7 @@ def enable_batch_invariant_mode():
|
||||
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":16:8"
|
||||
os.environ["CUBLASLT_WORKSPACE_SIZE"] = "1"
|
||||
|
||||
# Query the shared memory size and set block size
|
||||
# accordingly to avoid triton OutOfResources.
|
||||
if get_max_shared_memory_bytes() > 106496:
|
||||
_fp16_block_size_n = 256
|
||||
_fp32_block_size_n = 128
|
||||
_fp32_num_stages = 3
|
||||
else:
|
||||
_fp16_block_size_n = 128
|
||||
# SM89 N=1 fp32 128/stages=3 compiles to 131072B (>101376).
|
||||
# Only N=1 OORs; keep 128/stages=3 for every other N.
|
||||
_fp32_block_size_n = 32
|
||||
_fp32_num_stages = 2
|
||||
_resolve_block_sizes()
|
||||
elif current_platform.is_xpu():
|
||||
# Tensor descriptors need a global-memory allocator; must be set outside
|
||||
# torch.compile regions (triton.set_allocator modifies global state).
|
||||
@@ -1100,6 +1346,8 @@ def enable_batch_invariant_mode():
|
||||
_NUM_SMS = num_compute_units(0)
|
||||
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::mm.out", mm_out_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::addmm.out", addmm_out_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::matmul", matmul_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::linear", linear_batch_invariant, key)
|
||||
|
||||
@@ -1123,7 +1371,9 @@ def enable_batch_invariant_mode():
|
||||
_batch_invariant_LIB.impl(
|
||||
"aten::bmm", bmm_batch_invariant, key, allow_override=True
|
||||
)
|
||||
torch.bmm = bmm_batch_invariant
|
||||
_batch_invariant_LIB.impl(
|
||||
"aten::bmm.out", bmm_out_batch_invariant, key, allow_override=True
|
||||
)
|
||||
|
||||
reduced_precision_val = (
|
||||
(False, False) if is_torch_equal_or_newer("2.10.0") else False
|
||||
@@ -1145,28 +1395,32 @@ def override_envs_for_invariance():
|
||||
|
||||
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
|
||||
|
||||
# NCCL determinism settings. NCCL keeps the launch mode process-wide, so,
|
||||
# unlike the pins below, it also applies to communicators shared with other
|
||||
# processes.
|
||||
os.environ["NCCL_LAUNCH_MODE"] = "GROUP"
|
||||
pin_nccl_env(
|
||||
{
|
||||
"NCCL_COLLNET_ENABLE": "0",
|
||||
"NCCL_NVLS_ENABLE": "0",
|
||||
"NCCL_P2P_NET_DISABLE": "1",
|
||||
"NCCL_MIN_NCHANNELS": "1",
|
||||
"NCCL_MAX_NCHANNELS": "1",
|
||||
"NCCL_PROTO": "Simple",
|
||||
# NCCL >= 2.31 zero-fills the algorithm table of every collective
|
||||
# when NCCL_ALGO is set and re-enables only the named ones; together
|
||||
# with the NCCL_PROTO above, collectives not named here end up with
|
||||
# no algorithm and fail with ncclInvalidUsage. Re-enable Ring and
|
||||
# Tree for all collectives, then pin AllReduce to Tree.
|
||||
"NCCL_ALGO": "ring,tree;allreduce:tree",
|
||||
"NCCL_NTHREADS": "1",
|
||||
"NCCL_SOCKET_NTHREADS": "1",
|
||||
}
|
||||
)
|
||||
if current_platform.is_rocm():
|
||||
# The skinny GEMMs pick their split of K from the batch size.
|
||||
os.environ["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
|
||||
else:
|
||||
# NCCL determinism settings. NCCL keeps the launch mode process-wide, so,
|
||||
# unlike the pins below, it also applies to communicators shared with other
|
||||
# processes.
|
||||
os.environ["NCCL_LAUNCH_MODE"] = "GROUP"
|
||||
pin_nccl_env(
|
||||
{
|
||||
"NCCL_COLLNET_ENABLE": "0",
|
||||
"NCCL_NVLS_ENABLE": "0",
|
||||
"NCCL_P2P_NET_DISABLE": "1",
|
||||
"NCCL_MIN_NCHANNELS": "1",
|
||||
"NCCL_MAX_NCHANNELS": "1",
|
||||
"NCCL_PROTO": "Simple",
|
||||
# NCCL >= 2.31 zero-fills the algorithm table of every collective
|
||||
# when NCCL_ALGO is set and re-enables only the named ones; together
|
||||
# with the NCCL_PROTO above, collectives not named here end up with
|
||||
# no algorithm and fail with ncclInvalidUsage. Re-enable Ring and
|
||||
# Tree for all collectives, then pin AllReduce to Tree.
|
||||
"NCCL_ALGO": "ring,tree;allreduce:tree",
|
||||
"NCCL_NTHREADS": "1",
|
||||
"NCCL_SOCKET_NTHREADS": "1",
|
||||
}
|
||||
)
|
||||
|
||||
# torch.compile settings
|
||||
os.environ["VLLM_USE_AOT_COMPILE"] = "0"
|
||||
|
||||
@@ -218,6 +218,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.rocm import (
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
@@ -253,6 +254,20 @@ def _get_linear_backend(*, quantization: str) -> str:
|
||||
return config.kernel_config.linear_backend
|
||||
|
||||
|
||||
def _batch_invariant_fp8_kernel(kernel, config, quantization: str):
|
||||
can_implement, reason = is_supported_and_can_implement_kernel(kernel, config, None)
|
||||
if not can_implement:
|
||||
raise ValueError(f"Batch invariance on ROCm: {reason}")
|
||||
linear_backend = _get_linear_backend(quantization=quantization)
|
||||
if linear_backend not in ("auto", "triton"):
|
||||
logger.warning_once(
|
||||
"VLLM_BATCH_INVARIANT overrides --linear-backend=%s; "
|
||||
"using the Triton backend for deterministic execution.",
|
||||
linear_backend,
|
||||
)
|
||||
return kernel
|
||||
|
||||
|
||||
# Kernel classes covered by each --linear-backend value.
|
||||
_LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
"b12x": {
|
||||
@@ -723,6 +738,13 @@ def init_fp8_linear_kernel(
|
||||
)
|
||||
|
||||
if activation_quant_key.scale.group_shape.is_per_group():
|
||||
if envs.VLLM_BATCH_INVARIANT and current_platform.is_rocm():
|
||||
force_kernel = _batch_invariant_fp8_kernel(
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
scaled_mm_linear_kernel_config,
|
||||
"fp8_block_w8a8",
|
||||
)
|
||||
|
||||
kernel_type = choose_scaled_mm_linear_kernel(
|
||||
config=scaled_mm_linear_kernel_config,
|
||||
possible_kernels=_POSSIBLE_FP8_BLOCK_KERNELS, # type: ignore[misc]
|
||||
@@ -755,6 +777,13 @@ def init_fp8_linear_kernel(
|
||||
)
|
||||
|
||||
else:
|
||||
if envs.VLLM_BATCH_INVARIANT and current_platform.is_rocm():
|
||||
force_kernel = _batch_invariant_fp8_kernel(
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
scaled_mm_linear_kernel_config,
|
||||
"fp8_w8a8",
|
||||
)
|
||||
|
||||
kernel_type = choose_scaled_mm_linear_kernel(
|
||||
config=scaled_mm_linear_kernel_config,
|
||||
possible_kernels=_POSSIBLE_FP8_KERNELS, # type: ignore[arg-type]
|
||||
@@ -780,6 +809,19 @@ def init_fp8_linear_kernel(
|
||||
)
|
||||
|
||||
|
||||
def fp8_scaled_mm_is_batch_invariant(
|
||||
kernel: FP8ScaledMMLinearKernel | Fp8BlockScaledMMLinearKernel,
|
||||
) -> bool:
|
||||
"""Whether a row's output from the kernel is independent of the other rows."""
|
||||
if not isinstance(
|
||||
kernel, (CutlassFP8ScaledMMLinearKernel, TritonFP8ScaledMMLinearKernel)
|
||||
):
|
||||
return False
|
||||
# A dynamic per-tensor activation scale is an amax over every row.
|
||||
scale = kernel.config.activation_quant_key.scale
|
||||
return scale.static or not scale.group_shape.is_per_tensor()
|
||||
|
||||
|
||||
def init_int8_linear_kernel(
|
||||
is_channelwise: bool,
|
||||
is_static_input_scheme: bool,
|
||||
@@ -1250,6 +1292,7 @@ def register_linear_kernel(
|
||||
|
||||
|
||||
__all__ = [
|
||||
"fp8_scaled_mm_is_batch_invariant",
|
||||
"init_fp8_linear_kernel",
|
||||
"init_int8_linear_kernel",
|
||||
"init_nvfp4_linear_kernel",
|
||||
|
||||
@@ -62,7 +62,12 @@ if is_aiter_found_and_supported():
|
||||
N = weight.shape[0]
|
||||
K = weight.shape[1]
|
||||
if rocm_use_aiter_fp4_asm_gemm:
|
||||
if M <= 64 and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K):
|
||||
# The preshuffle kernel's tuned configs split K in some M buckets.
|
||||
if (
|
||||
M <= 64
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K)
|
||||
):
|
||||
if x_scales is None:
|
||||
# use hip quant kernel for performance
|
||||
if M >= 32:
|
||||
@@ -116,7 +121,16 @@ if is_aiter_found_and_supported():
|
||||
x_q.shape[0], weight.shape[0], device=x_q.device, dtype=out_dtype
|
||||
)
|
||||
|
||||
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y)
|
||||
config = None
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
# The configs split K in some M buckets (up to M <= 256 for
|
||||
# tuned shapes). Keep the tile, drop the split.
|
||||
from aiter.ops.triton.gemm_afp4wfp4 import _get_config
|
||||
|
||||
config, _ = _get_config(M, N, K)
|
||||
config = dict(config, NUM_KSPLIT=1)
|
||||
|
||||
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y, config)
|
||||
return y
|
||||
|
||||
def gemm_with_dynamic_quant_fake(
|
||||
|
||||
@@ -12,6 +12,7 @@ instead, since ``can_implement`` does not filter on K.
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
@@ -246,7 +247,10 @@ class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
N, K = weight.shape
|
||||
scale_k = K // MXFP8_BLOCK_SIZE
|
||||
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
|
||||
block_scale = _as_block32_scale(weight_scale)
|
||||
# rocm_mxfp8_block32_gemm's split-K varies with M.
|
||||
block_scale = (
|
||||
None if envs.VLLM_BATCH_INVARIANT else _as_block32_scale(weight_scale)
|
||||
)
|
||||
if block_scale is not None:
|
||||
# Checkpoints with 32x32 scale blocks (DeepSeek V4/V4.1) get them
|
||||
# back from the loader expanded per row; keep the blocks and use
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
|
||||
ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
@@ -65,6 +66,7 @@ __all__ = [
|
||||
"PerTensorTorchFP8ScaledMMLinearKernel",
|
||||
"RowWiseTorchFP8ScaledMMLinearKernel",
|
||||
"ROCmFP8ScaledMMLinearKernel",
|
||||
"TritonFP8ScaledMMLinearKernel",
|
||||
"TritonInt8ScaledMMLinearKernel",
|
||||
"ZentorchInt8ScaledMMLinearKernel",
|
||||
"Fp8BlockScaledMMLinearKernel",
|
||||
|
||||
@@ -20,6 +20,8 @@ from .BlockScaledMMLinearKernel import (
|
||||
)
|
||||
from .cutlass import CutlassInt8ScaledMMLinearKernel
|
||||
from .ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearKernel,
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
Int8ScaledMMLinearLayerConfig,
|
||||
)
|
||||
|
||||
@@ -156,6 +158,46 @@ class TritonInt8ScaledMMLinearKernel(CutlassInt8ScaledMMLinearKernel):
|
||||
return out
|
||||
|
||||
|
||||
class TritonFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_cuda_alike():
|
||||
return False, "requires ROCm or CUDA."
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
act_group = c.activation_quant_key.scale.group_shape
|
||||
weight_group = c.weight_quant_key.scale.group_shape
|
||||
if not (act_group.is_per_tensor() or act_group.is_per_token()):
|
||||
return False, "requires per-tensor or per-token activation scales."
|
||||
if not (
|
||||
weight_group.is_per_tensor()
|
||||
or weight_group.is_per_channel()
|
||||
or weight_group.is_per_token()
|
||||
):
|
||||
return False, "requires per-tensor, per-channel or per-token weight scales."
|
||||
return True, None
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
out = triton_scaled_mm(
|
||||
A, B, scale_a=As, scale_b=Bs, out_dtype=out_dtype, bias=bias
|
||||
)
|
||||
return out.view(*output_shape)
|
||||
|
||||
|
||||
class TritonFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(cls, compute_capability=None):
|
||||
|
||||
@@ -51,6 +51,10 @@ class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
|
||||
def _supports_parallel_config(moe_parallel_config) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_batch_invariance() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
|
||||
@@ -218,7 +218,7 @@ class OCP_MXQuantizationEmulationTritonExperts(TritonExperts):
|
||||
global_num_experts=global_num_experts,
|
||||
expert_map=expert_map,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
a2_scale=a2_scale,
|
||||
workspace13=workspace13,
|
||||
workspace2=workspace2,
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
|
||||
@@ -150,16 +150,16 @@ def scaled_mm_kernel(
|
||||
scale_b = scale_b.broadcast_to((BLOCK_SIZE_N, 1))
|
||||
accumulator = scale_b.T * accumulator.to(tl.float32)
|
||||
|
||||
# Convert to output format.
|
||||
c = accumulator.to(c_ptr.type.element_ty)
|
||||
|
||||
# Add bias, it's already in output format, so add it after conversion.
|
||||
# Add bias in accumulator dtype for less rounding error.
|
||||
if bias_ptr:
|
||||
offsets_bias = offsets_bn
|
||||
bias_ptrs = bias_ptr + offsets_bias
|
||||
bias_mask = offsets_bias < N
|
||||
bias = tl.load(bias_ptrs, bias_mask)
|
||||
c += bias
|
||||
accumulator += bias
|
||||
|
||||
# Convert to output format.
|
||||
c = accumulator.to(c_ptr.type.element_ty)
|
||||
|
||||
# Save output
|
||||
offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64)
|
||||
@@ -224,6 +224,8 @@ def triton_scaled_mm(
|
||||
tile_shape = (64, 128, 128)
|
||||
else:
|
||||
tile_shape = (128, 128, 128)
|
||||
else:
|
||||
tile_shape = (block_size_m, block_size_n, block_size_k)
|
||||
|
||||
block_size_m, block_size_n, block_size_k = tile_shape
|
||||
|
||||
|
||||
@@ -9,10 +9,10 @@ import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
fp8_scaled_mm_is_batch_invariant,
|
||||
init_fp8_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
@@ -432,7 +432,7 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
bias,
|
||||
)
|
||||
else:
|
||||
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
|
||||
if fp8_scaled_mm_is_batch_invariant(self.fp8_linear):
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
# per-tensor/channel: dequant to BF16 and run GEMM
|
||||
|
||||
@@ -15,9 +15,11 @@ if TYPE_CHECKING:
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
fp8_scaled_mm_is_batch_invariant,
|
||||
init_fp8_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
@@ -278,7 +280,7 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
) -> torch.Tensor:
|
||||
# if batch invariant mode is enabled, use BF16 dequant
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
|
||||
if fp8_scaled_mm_is_batch_invariant(self.fp8_linear):
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
weight_fp8 = layer.weight.to(torch.bfloat16)
|
||||
@@ -466,7 +468,7 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
# if batch invariant mode is enabled dequant
|
||||
if (
|
||||
envs.VLLM_BATCH_INVARIANT
|
||||
and not isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel)
|
||||
and not fp8_scaled_mm_is_batch_invariant(self.fp8_linear)
|
||||
and not isinstance(x, QuantizedActivation)
|
||||
):
|
||||
weight_dequant = (
|
||||
|
||||
@@ -1041,7 +1041,10 @@ def w8a8_triton_block_scaled_mm(
|
||||
C_shape = A.shape[:-1] + (N,)
|
||||
C = A.new_empty(C_shape, dtype=output_dtype)
|
||||
|
||||
configs = get_w8a8_block_fp8_configs(N, K, block_size[0], block_size[1])
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
configs = None # use the default config
|
||||
else:
|
||||
configs = get_w8a8_block_fp8_configs(N, K, block_size[0], block_size[1])
|
||||
if configs:
|
||||
# Get the optimal config if there is one
|
||||
config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
|
||||
|
||||
@@ -288,6 +288,13 @@ def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, boo
|
||||
def rocm_unquantized_gemm_impl(
|
||||
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
from vllm.model_executor.determinism.batch_invariant import (
|
||||
linear_batch_invariant,
|
||||
)
|
||||
|
||||
return linear_batch_invariant(x, weight, bias)
|
||||
|
||||
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx950, on_gfx1250
|
||||
|
||||
n = x.numel() // x.size(-1)
|
||||
|
||||
Reference in New Issue
Block a user