[ROCm] Fix MoE kernel test failures on gfx950 (#37833)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
co-authored by
Woosuk Kwon
Matthew Wong
parent
e38817fadb
commit
7d6917bef5
@@ -741,11 +741,14 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
# TP=4 yields intermediate_size_per_partition=384), AITER raises:
|
||||
# "device_gemm ... does not support this GEMM problem".
|
||||
# Fall back to emulation in that case.
|
||||
# For gpt_oss models, create_weights rounds up the dimensions
|
||||
# internally, so the alignment check is skipped.
|
||||
if (
|
||||
not self.emulate
|
||||
and self.use_rocm_aiter_moe
|
||||
and self.ocp_mx_scheme is not None
|
||||
and self.ocp_mx_scheme.startswith("w_mxfp4")
|
||||
and self.model_type != "gpt_oss"
|
||||
and moe.intermediate_size_per_partition % CK_MXFP4_MOE_DIM_ALIGNMENT != 0
|
||||
):
|
||||
logger.warning_once(
|
||||
@@ -819,6 +822,18 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
"unpadded_hidden_size", hidden_size
|
||||
)
|
||||
|
||||
# On GFX950, the GFX950MXScaleLayout swizzle requires
|
||||
# hidden_size to be a multiple of 256 (SCALE_K = hidden_size / 32
|
||||
# must be divisible by 8). Pad hidden_size for weight/scale
|
||||
# allocation; the original value is preserved in unpadded_hidden_size.
|
||||
# Only applies to the native (non-emulated) CK path on GFX950.
|
||||
if (
|
||||
self.model_type == "gpt_oss"
|
||||
and current_platform.is_rocm()
|
||||
and not self.emulate
|
||||
):
|
||||
hidden_size = round_up(hidden_size, 256)
|
||||
|
||||
# WEIGHTS
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
|
||||
@@ -615,8 +615,8 @@ def _per_token_group_quant_fp8(
|
||||
# Avoid to divide zero
|
||||
eps,
|
||||
# Information for float8
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
fp8_min: tl.constexpr,
|
||||
fp8_max: tl.constexpr,
|
||||
use_ue8m0: tl.constexpr,
|
||||
# Meta-parameters
|
||||
BLOCK: tl.constexpr,
|
||||
@@ -647,8 +647,12 @@ def _per_token_group_quant_fp8(
|
||||
|
||||
y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
# Quant
|
||||
# Use multiply-by-reciprocal instead of division to match PyTorch's
|
||||
# tensor/scalar division precision (GPU fast-division for constexpr
|
||||
# divisors can introduce 1-ULP error that flips FP8 quantization at
|
||||
# representable-value boundaries).
|
||||
_absmax = tl.maximum(tl.max(tl.abs(y)), eps)
|
||||
scale_raw = _absmax / fp8_max
|
||||
scale_raw = _absmax * (1.0 / fp8_max)
|
||||
y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw
|
||||
y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
|
||||
|
||||
@@ -667,8 +671,8 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
y_s_col_stride: tl.int64,
|
||||
# Information for float8
|
||||
eps,
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
fp8_min: tl.constexpr,
|
||||
fp8_max: tl.constexpr,
|
||||
use_ue8m0: tl.constexpr,
|
||||
# Meta-parameters
|
||||
GROUP_SIZE: tl.constexpr,
|
||||
@@ -709,7 +713,7 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
|
||||
# quant
|
||||
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
|
||||
scale_raw = _absmax / fp8_max
|
||||
scale_raw = _absmax * (1.0 / fp8_max)
|
||||
y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw
|
||||
y_s = tl.reshape(y_s, (BLOCK_M, 1))
|
||||
y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
|
||||
@@ -808,8 +812,8 @@ def _per_token_group_quant_fp8_colmajor(
|
||||
# Avoid to divide zero
|
||||
eps,
|
||||
# Information for float8
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
fp8_min: tl.constexpr,
|
||||
fp8_max: tl.constexpr,
|
||||
use_ue8m0: tl.constexpr,
|
||||
# Meta-parameters
|
||||
BLOCK: tl.constexpr,
|
||||
@@ -849,7 +853,7 @@ def _per_token_group_quant_fp8_colmajor(
|
||||
y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
# Quant
|
||||
_absmax = tl.maximum(tl.max(tl.abs(y)), eps)
|
||||
scale_raw = _absmax / fp8_max
|
||||
scale_raw = _absmax * (1.0 / fp8_max)
|
||||
y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw
|
||||
y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user