[ROCm][Perf] Add bpreshuffled blockscaled fp8 GEMM (#51692)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
This commit is contained in:
+27
-5
@@ -175,18 +175,26 @@ def is_aiter_found_and_supported_on_rdna4() -> bool:
|
||||
|
||||
@functools.cache
|
||||
def _load_gemm_tuned_configs(
|
||||
q_dtype_w: torch.dtype, csv_path: str
|
||||
) -> set[tuple[int, int, int]]:
|
||||
csv_path: str,
|
||||
filters: tuple[tuple[str, object], ...],
|
||||
key_cols: tuple[str, ...] = ("N", "K", "M"),
|
||||
) -> set[tuple[int, ...]]:
|
||||
try:
|
||||
df = pd.read_csv(csv_path).drop_duplicates()
|
||||
df = df[df["q_dtype_w"] == str(q_dtype_w)]
|
||||
return set(zip(df["N"].astype(int), df["K"].astype(int), df["M"].astype(int)))
|
||||
for col, val in filters:
|
||||
if col not in df.columns:
|
||||
continue
|
||||
if isinstance(val, int):
|
||||
df = df[df[col].astype(int) == val]
|
||||
else:
|
||||
df = df[df[col].astype(str) == str(val)]
|
||||
return set(zip(*(df[c].astype(int) for c in key_cols)))
|
||||
except Exception:
|
||||
return set()
|
||||
|
||||
|
||||
def _check_kernel_tuned(N: int, K: int, q_dtype_w: torch.dtype, csv_path: str) -> bool:
|
||||
configs = _load_gemm_tuned_configs(q_dtype_w, csv_path)
|
||||
configs = _load_gemm_tuned_configs(csv_path, (("q_dtype_w", q_dtype_w),))
|
||||
l_m = (
|
||||
[1, 2, 4]
|
||||
+ list(range(8, 513, 8))
|
||||
@@ -3200,6 +3208,20 @@ class rocm_aiter_ops:
|
||||
csv_path = aiter_gemm_a8w8_ops.AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_FILE
|
||||
return _check_kernel_tuned(N, K, q_dtype_w, csv_path)
|
||||
|
||||
@staticmethod
|
||||
def is_blockscale_bpreshuffle_tuned(n: int, k: int) -> bool:
|
||||
"""Whether (N, K) has a tuned aiter blockscale bpreshuffle config."""
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
import aiter.ops.gemm_op_a8w8 as aiter_gemm_a8w8_ops
|
||||
|
||||
csv_path = aiter_gemm_a8w8_ops.AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE_FILE
|
||||
gfx = aiter_gemm_a8w8_ops.get_gfx()
|
||||
cu_num = aiter_gemm_a8w8_ops.get_cu_num()
|
||||
return (n, k) in _load_gemm_tuned_configs(
|
||||
csv_path, (("gfx", gfx), ("cu_num", cu_num)), key_cols=("N", "K")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def shuffle_weight(
|
||||
tensor: torch.Tensor, layout: tuple[int, int] = (16, 16)
|
||||
|
||||
@@ -178,6 +178,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.aiter import (
|
||||
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
|
||||
AiterInt8ScaledMMLinearKernel,
|
||||
AiterPerTokenFp8ScaledMMLinearKernel,
|
||||
AiterPreshuffledFp8BlockScaledMMKernel,
|
||||
AiterPreshuffledPerTokenFp8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.b12x import (
|
||||
@@ -316,6 +317,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
"aiter": {
|
||||
AiterInt8ScaledMMLinearKernel,
|
||||
AiterFp8BlockScaledMMKernel,
|
||||
AiterPreshuffledFp8BlockScaledMMKernel,
|
||||
AiterPerTokenFp8ScaledMMLinearKernel,
|
||||
AiterPreshuffledPerTokenFp8ScaledMMLinearKernel,
|
||||
AiterMxfp4LinearKernel,
|
||||
@@ -456,6 +458,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
|
||||
BlockWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
AiterPreshuffledFp8BlockScaledMMKernel,
|
||||
AiterFp8BlockScaledMMKernel,
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
],
|
||||
@@ -1210,6 +1213,7 @@ __all__ = [
|
||||
"ScaledMMLinearLayerConfig",
|
||||
"AiterHipbMMPerTokenFp8ScaledMMLinearKernel",
|
||||
"AiterPreshuffledPerTokenFp8ScaledMMLinearKernel",
|
||||
"AiterPreshuffledFp8BlockScaledMMKernel",
|
||||
"AiterPerTokenFp8ScaledMMLinearKernel",
|
||||
"NvFp4LinearKernel",
|
||||
"NvFp4LinearLayerConfig",
|
||||
|
||||
@@ -457,3 +457,131 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
return gemm_a8w8_blockscale_op(
|
||||
A, B, As, Bs, list(self.weight_group_shape), output_dtype=out_dtype
|
||||
)
|
||||
|
||||
|
||||
class AiterPreshuffledFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
"""Aiter FP8 block-scaled GEMM using a pre-shuffled (bpreshuffle) weight."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
return AiterPreshuffledPerTokenFp8ScaledMMLinearKernel.is_supported(
|
||||
compute_capability
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: FP8ScaledMMLinearLayerConfig):
|
||||
can_implement_base, reason = super().can_implement(config)
|
||||
if not can_implement_base:
|
||||
return can_implement_base, reason
|
||||
|
||||
act_quant_desc = config.activation_quant_key.scale
|
||||
if act_quant_desc.group_shape != GroupShape(1, 128):
|
||||
return (
|
||||
False,
|
||||
(
|
||||
"Supports only dynamic per token group activation "
|
||||
"quantization with group_shape=(1,128)."
|
||||
),
|
||||
)
|
||||
|
||||
# bpreshuffle GEMM requires aiter fp8 linear to be enabled and an fp8
|
||||
# 2D weight with N and K divisible by 128. Require a tuned config as the
|
||||
# fallback can trigger faults or numerical errors.
|
||||
if not rocm_aiter_ops.is_linear_fp8_enabled():
|
||||
return (
|
||||
False,
|
||||
(
|
||||
"requires setting `VLLM_ROCM_USE_AITER=1` "
|
||||
"and `VLLM_ROCM_USE_AITER_LINEAR=1`."
|
||||
),
|
||||
)
|
||||
|
||||
n, k = config.weight_shape
|
||||
if not (n % 128 == 0 and k % 128 == 0):
|
||||
return (
|
||||
False,
|
||||
(
|
||||
f"requires N and K dimensions divisible by 128, received "
|
||||
f"N={n} and K={k}."
|
||||
),
|
||||
)
|
||||
|
||||
if not rocm_aiter_ops.is_blockscale_bpreshuffle_tuned(n, k):
|
||||
return (
|
||||
False,
|
||||
(
|
||||
f"requires a tuned aiter blockscale bpreshuffle config for "
|
||||
f"N={n} and K={k}."
|
||||
),
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
super().process_weights_after_loading(layer)
|
||||
|
||||
params = FP8BlockParams.from_layer(layer)
|
||||
if params.weight_scale_inv is not None:
|
||||
ws, attr = params.weight_scale_inv, params.WEIGHT_SCALE_INV
|
||||
else:
|
||||
ws, attr = params.weight_scale, params.WEIGHT_SCALE
|
||||
if ws is not None and ws.dtype == torch.float8_e8m0fnu:
|
||||
replace_parameter(layer, attr, _upcast_e8m0_to_fp32(ws).contiguous())
|
||||
|
||||
weight = params.weight
|
||||
# runtime safety net
|
||||
assert (
|
||||
weight.dim() == 2
|
||||
and weight.dtype == current_platform.fp8_dtype()
|
||||
and weight.shape[0] % 128 == 0
|
||||
and weight.shape[1] % 128 == 0
|
||||
), (
|
||||
"AiterPreshuffledFp8BlockScaledMMKernel requires a 2D fp8 weight "
|
||||
"with N and K divisible by 128."
|
||||
)
|
||||
|
||||
shuffled_weight = rocm_aiter_ops.shuffle_weight(
|
||||
weight.contiguous(), layout=(16, 16)
|
||||
)
|
||||
replace_parameter(
|
||||
layer,
|
||||
params.WEIGHT,
|
||||
torch.nn.Parameter(shuffled_weight.data, requires_grad=False),
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
params = self._get_layer_params(layer)
|
||||
Bs = (
|
||||
params.weight_scale
|
||||
if params.weight_scale_inv is None
|
||||
else params.weight_scale_inv
|
||||
)
|
||||
|
||||
x_2d = x.view(-1, x.shape[-1])
|
||||
A, As = rocm_aiter_ops.group_fp8_quant(x_2d, transpose_scale=True)
|
||||
output = rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle(
|
||||
A, params.weight, As, Bs, output_dtype=self.config.out_dtype
|
||||
)
|
||||
if bias is not None:
|
||||
output = output + bias
|
||||
return output.view(*x.shape[:-1], params.weight.shape[0])
|
||||
|
||||
def apply_block_scaled_mm(
|
||||
self,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError(
|
||||
"AiterPreshuffledFp8BlockScaledMMKernel overrides apply_weights and "
|
||||
"does not use apply_block_scaled_mm."
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user