[ROCm] gfx1250 optimizations and model enablement (#58800)
Signed-off-by: jpvillam <juan.villamizar@amd.com> Signed-off-by: Jaden Mathias <jaden.mathias@amd.com> Signed-off-by: Juan Villamizar <jpvillam@amd.com> Signed-off-by: jpvillam <jpvillam@amd.com> Signed-off-by: Omar Muhammad <omar.muhammad@amd.com> Signed-off-by: Daniel <danichan@amd.com> Signed-off-by: Juan Villamizar <100237675+jpvillam-amd@users.noreply.github.com> Co-authored-by: Jaden Mathias <jaden.mathias@amd.com> Co-authored-by: jpvillam <jpvillam@amd.com> Co-authored-by: Omar Muhammad <omar.muhammad@amd.com> Co-authored-by: Daniel <danichan@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Daniel Chan <Daniel.Chan@amd.com>
This commit is contained in:
co-authored by
Jaden Mathias
jpvillam
Omar Muhammad
Daniel
Claude Opus 5
Daniel Chan
parent
9b99ccb56b
commit
e711723352
@@ -9,6 +9,7 @@ from vllm._aiter_ops import (
|
||||
rocm_aiter_ops,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
)
|
||||
@@ -399,6 +400,26 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
rocm_aiter_ops.is_triton_gemm_w8a8_tuned(n, k) or _on_gfx1250
|
||||
)
|
||||
|
||||
self.use_bpreshuffle = (
|
||||
_on_gfx1250
|
||||
and n % 16 == 0
|
||||
and k % 128 == 0
|
||||
and bool(rocm_aiter_ops.is_triton_blockscale_bpreshuffle_gemm_tuned(n, k))
|
||||
)
|
||||
if self.use_bpreshuffle:
|
||||
# The tuned preshuffle configs expect a column-major group scale.
|
||||
act_scale_descriptor = config.activation_quant_key.scale
|
||||
self.quant_fp8 = QuantFP8(
|
||||
static=act_scale_descriptor.static,
|
||||
group_shape=act_scale_descriptor.group_shape,
|
||||
num_token_padding=self.get_output_padding(),
|
||||
use_ue8m0=False,
|
||||
column_major_scales=True,
|
||||
)
|
||||
logger.info_once(
|
||||
"Using aiter A8W8 blockscale B-preshuffle GEMM for N=%d K=%d", n, k
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
super().process_weights_after_loading(layer)
|
||||
|
||||
@@ -410,6 +431,18 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
if ws is not None and ws.dtype == torch.float8_e8m0fnu:
|
||||
replace_parameter(layer, attr, _upcast_e8m0_to_fp32(ws).contiguous())
|
||||
|
||||
if getattr(layer, "is_bmm", False):
|
||||
self.use_bpreshuffle = False
|
||||
|
||||
if self.use_bpreshuffle:
|
||||
w = FP8BlockParams.from_layer(layer).weight
|
||||
replace_parameter(
|
||||
layer,
|
||||
FP8BlockParams.WEIGHT,
|
||||
rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)),
|
||||
)
|
||||
layer.aiter_bpreshuffled = True
|
||||
|
||||
@classmethod
|
||||
def is_supported(cls, compute_capability=None):
|
||||
if (
|
||||
@@ -464,7 +497,11 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
Bs = Bs.to(torch.float32)
|
||||
|
||||
out_dtype = self.config.out_dtype
|
||||
if self.use_triton:
|
||||
if self.use_bpreshuffle:
|
||||
gemm_a8w8_blockscale_op = (
|
||||
rocm_aiter_ops.triton_gemm_a8w8_blockscale_preshuffle
|
||||
)
|
||||
elif self.use_triton:
|
||||
gemm_a8w8_blockscale_op = rocm_aiter_ops.triton_gemm_a8w8_blockscale
|
||||
else:
|
||||
gemm_a8w8_blockscale_op = rocm_aiter_ops.gemm_a8w8_blockscale
|
||||
|
||||
Reference in New Issue
Block a user