[ROCm] gfx1250 optimizations and model enablement (#58800)

Signed-off-by: jpvillam <juan.villamizar@amd.com>
Signed-off-by: Jaden Mathias <jaden.mathias@amd.com>
Signed-off-by: Juan Villamizar <jpvillam@amd.com>
Signed-off-by: jpvillam <jpvillam@amd.com>
Signed-off-by: Omar Muhammad <omar.muhammad@amd.com>
Signed-off-by: Daniel <danichan@amd.com>
Signed-off-by: Juan Villamizar <100237675+jpvillam-amd@users.noreply.github.com>
Co-authored-by: Jaden Mathias <jaden.mathias@amd.com>
Co-authored-by: jpvillam <jpvillam@amd.com>
Co-authored-by: Omar Muhammad <omar.muhammad@amd.com>
Co-authored-by: Daniel <danichan@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Daniel Chan <Daniel.Chan@amd.com>
This commit is contained in:
Juan Villamizar
2026-10-15 14:51:57 -07:00
committed by GitHub
co-authored by Jaden Mathias jpvillam Omar Muhammad Daniel Claude Opus 5 Daniel Chan
parent 9b99ccb56b
commit e711723352
21 changed files with 845 additions and 73 deletions
@@ -9,6 +9,7 @@ from vllm._aiter_ops import (
rocm_aiter_ops,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_upcast_e8m0_to_fp32,
)
@@ -399,6 +400,26 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
rocm_aiter_ops.is_triton_gemm_w8a8_tuned(n, k) or _on_gfx1250
)
self.use_bpreshuffle = (
_on_gfx1250
and n % 16 == 0
and k % 128 == 0
and bool(rocm_aiter_ops.is_triton_blockscale_bpreshuffle_gemm_tuned(n, k))
)
if self.use_bpreshuffle:
# The tuned preshuffle configs expect a column-major group scale.
act_scale_descriptor = config.activation_quant_key.scale
self.quant_fp8 = QuantFP8(
static=act_scale_descriptor.static,
group_shape=act_scale_descriptor.group_shape,
num_token_padding=self.get_output_padding(),
use_ue8m0=False,
column_major_scales=True,
)
logger.info_once(
"Using aiter A8W8 blockscale B-preshuffle GEMM for N=%d K=%d", n, k
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
super().process_weights_after_loading(layer)
@@ -410,6 +431,18 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
if ws is not None and ws.dtype == torch.float8_e8m0fnu:
replace_parameter(layer, attr, _upcast_e8m0_to_fp32(ws).contiguous())
if getattr(layer, "is_bmm", False):
self.use_bpreshuffle = False
if self.use_bpreshuffle:
w = FP8BlockParams.from_layer(layer).weight
replace_parameter(
layer,
FP8BlockParams.WEIGHT,
rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)),
)
layer.aiter_bpreshuffled = True
@classmethod
def is_supported(cls, compute_capability=None):
if (
@@ -464,7 +497,11 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
Bs = Bs.to(torch.float32)
out_dtype = self.config.out_dtype
if self.use_triton:
if self.use_bpreshuffle:
gemm_a8w8_blockscale_op = (
rocm_aiter_ops.triton_gemm_a8w8_blockscale_preshuffle
)
elif self.use_triton:
gemm_a8w8_blockscale_op = rocm_aiter_ops.triton_gemm_a8w8_blockscale
else:
gemm_a8w8_blockscale_op = rocm_aiter_ops.gemm_a8w8_blockscale