Enable gfx1250 ROCm architecture (#46516)

Signed-off-by: Douglas Lehr <Doug.Lehr@amd.com>
Signed-off-by: jpvillam <juan.villamizar@amd.com>
Signed-off-by: Daniel <danichan@amd.com>
Signed-off-by: Jaden Mathias <jaden.mathias@amd.com>
Signed-off-by: jpvillam <jpvillam@amd.com>
Signed-off-by: Juan Villamizar <100237675+jpvillam-amd@users.noreply.github.com>
Signed-off-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Signed-off-by: Jaden Mathias <Jaden.Mathias@amd.com>
Co-authored-by: Douglas Lehr <Doug.Lehr@amd.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-authored-by: Daniel <danichan@amd.com>
Co-authored-by: Jaden Mathias <jaden.mathias@amd.com>
Co-authored-by: jpvillam <jpvillam@amd.com>
Co-authored-by: root <root@ctheliosr-1b114-f01-2.mnb.dcgpu>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: dllehr <douglehr@amd.com>
Co-authored-by: Daniel Chan <Daniel.Chan@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Juan Villamizar
2026-07-31 01:41:12 +00:00
committed by GitHub
co-authored by Douglas Lehr Claude Opus 4.7 Daniel Jaden Mathias jpvillam root Douglas Lehr dllehr Daniel Chan Andreas Karatzas
parent 553fcb82d5
commit 4f1da84eb5
33 changed files with 1850 additions and 104 deletions
@@ -20,7 +20,9 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
__all__ = [
"AiterW4A8ExpertsMonolithic",
"AiterW4A16ExpertsMonolithic",
"aiter_triton_kernel_w4a8_moe_forward",
"aiter_triton_kernel_w4a16_moe_forward",
]
@@ -46,11 +48,27 @@ def aiter_triton_kernel_w4a8_moe_forward(
and quant_config.use_mxfp4_w4a8
and rocm_aiter_ops.is_enabled()
)
from aiter.ops.triton.moe_routing.routing import routing as aiter_routing
from vllm.platforms.rocm import on_gfx1250
try:
from aiter.ops.triton.moe.moe_routing import routing as _routing_mod
except ImportError:
from aiter.ops.triton.moe_routing import routing as _routing_mod
if on_gfx1250():
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output, topk, sm_first=not renormalize
)
# gfx1250: aiter's in-kernel gather is numerically broken
if on_gfx1250():
gather_src = gather_idx.to(torch.long) // topk
hidden_states = hidden_states[gather_src]
gather_idx = None
return triton_kernel_fused_mxfp4_w4a8_experts(
None,
hidden_states,
@@ -199,12 +217,11 @@ class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
@staticmethod
def _supports_current_device() -> bool:
# Requires AITER and GFX950
if not rocm_aiter_ops.is_enabled():
return False
from vllm.platforms.rocm import on_gfx950
from vllm.platforms.rocm import on_gfx950, on_gfx1250
return on_gfx950()
return on_gfx950() or on_gfx1250()
@staticmethod
def _supports_no_act_and_mul() -> bool:
@@ -293,3 +310,358 @@ class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
unpadded_N_w2=self.moe_config.hidden_dim_unpadded,
unpadded_K_w2=self.moe_config.intermediate_size_per_partition_unpadded,
)
def _aiter_raw(t):
if t is None or isinstance(t, torch.Tensor):
return t
return t.storage.data if hasattr(t, "storage") else t
def _aiter_w4a16_silu_via_a8w4(
hidden_states: torch.Tensor,
w1_data,
w2_data,
w1_wscale,
w2_wscale,
w1_bias,
w2_bias,
routing_data,
gather_idx,
scatter_idx,
gammas,
apply_router_weight_on_input: bool,
swiglu_limit: float,
unpadded_N_w1,
unpadded_K_w1,
unpadded_N_w2,
unpadded_K_w2,
) -> torch.Tensor:
"""
MXFP4 w4a16 MoE with a SILU (concatenated ``[gate | up]``) activation.
"""
from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul
from aiter.ops.triton.moe_op_gemm_a8w4 import moe_gemm_a8w4
from aiter.ops.triton.quant import dynamic_mxfp8_quant
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
should_use_cdna4_mx_scale_swizzle,
)
swz = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None
quant_dtype = torch.float8_e4m3fn
g1_gammas = gammas if apply_router_weight_on_input else None
g2_gammas = None if apply_router_weight_on_input else gammas
hidden_q, a1_scale = dynamic_mxfp8_quant(hidden_states, quant_dtype=quant_dtype)
raw_gate_up = moe_gemm_a8w4(
hidden_q,
w1_data,
a1_scale,
w1_wscale,
None,
None,
w1_bias,
routing_data,
gather_indx=gather_idx,
gammas=g1_gammas,
swizzle_mx_scale=swz,
out_dtype=torch.bfloat16,
apply_swiglu=False,
unpadded_N=unpadded_N_w1,
unpadded_K=unpadded_K_w1,
)
if unpadded_N_w1 is not None:
raw_gate_up = raw_gate_up[:, :unpadded_N_w1]
interim_fp8, a2_scale = fused_clamp_act_mul(
raw_gate_up,
swiglu_limit=swiglu_limit,
activation="silu",
dtype_quant=quant_dtype,
scale_dtype_fmt="ue8m0",
quant_block_size=32,
)
out = moe_gemm_a8w4(
interim_fp8,
w2_data,
a2_scale,
w2_wscale,
None,
None,
w2_bias,
routing_data,
scatter_indx=scatter_idx,
gammas=g2_gammas,
swizzle_mx_scale=swz,
unpadded_N=unpadded_N_w2,
unpadded_K=unpadded_K_w2,
)
return out
def aiter_triton_kernel_w4a16_moe_forward(
hidden_states: torch.Tensor,
w1,
w2,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
activation: MoEActivation = MoEActivation.SWIGLUOAI,
quant_config: FusedMoEQuantConfig | None = None,
apply_router_weight_on_input: bool = False,
global_num_experts: int = -1,
expert_map: torch.Tensor | None = None,
unpadded_N_w1=None,
unpadded_K_w1=None,
unpadded_N_w2=None,
unpadded_K_w2=None,
num_expert_group: int | None = None,
topk_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
score_mode: str | None = None,
):
assert quant_config is not None and rocm_aiter_ops.is_enabled()
from vllm.platforms.rocm import on_gfx1250
try:
from aiter.ops.triton.moe.moe_op_gemm_a16w4 import moe_gemm_a16w4
from aiter.ops.triton.moe.moe_routing import routing as _routing_mod
except ImportError:
from aiter.ops.triton.moe.moe_op_gemm_a16w4 import moe_gemm_a16w4
from aiter.ops.triton.moe_routing import routing as _routing_mod
if on_gfx1250():
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
if score_mode is not None:
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output,
topk,
score_mode=score_mode,
bias=e_score_correction_bias,
renorm=renormalize,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
)
else:
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output, topk, sm_first=not renormalize
)
if on_gfx1250():
gather_src = gather_idx.to(torch.long) // topk
hidden_states = hidden_states[gather_src]
gather_idx = None
assert quant_config.w1_precision is not None
assert quant_config.w2_precision is not None
w1_data = _aiter_raw(w1)
w2_data = _aiter_raw(w2)
w1_wscale = _aiter_raw(quant_config.w1_precision.weight_scale)
w2_wscale = _aiter_raw(quant_config.w2_precision.weight_scale)
gammas = routing_data.gate_scal if routing_data else None
swiglu_alpha = (
quant_config.gemm1_alpha if quant_config.gemm1_alpha is not None else 1.0
)
swiglu_limit = (
quant_config.gemm1_clamp_limit
if quant_config.gemm1_clamp_limit is not None
else 7.0
)
# SILU on gfx1250: use the verified a8w4 kernel (dynamic MXFP8); a16w4 faults.
if activation == MoEActivation.SILU and on_gfx1250():
return _aiter_w4a16_silu_via_a8w4(
hidden_states,
w1_data,
w2_data,
w1_wscale,
w2_wscale,
quant_config.w1_bias,
quant_config.w2_bias,
routing_data,
gather_idx,
scatter_idx,
gammas,
apply_router_weight_on_input,
swiglu_limit,
unpadded_N_w1,
unpadded_K_w1,
unpadded_N_w2,
unpadded_K_w2,
)
# SILU: silu(gate) * up — same kernel, just no "+1" residual in swiglu.
swiglu_add_residual = activation != MoEActivation.SILU
intermediate = moe_gemm_a16w4(
hidden_states,
w1_data,
None,
w1_wscale,
None,
None,
quant_config.w1_bias,
routing_data,
gather_indx=gather_idx,
gammas=gammas if apply_router_weight_on_input else None,
swizzle_mx_scale=None,
apply_swiglu=True,
alpha=swiglu_alpha,
limit=swiglu_limit,
swiglu_add_residual=swiglu_add_residual,
unpadded_N=unpadded_N_w1,
unpadded_K=unpadded_K_w1,
)
out = moe_gemm_a16w4(
intermediate,
w2_data,
None,
w2_wscale,
None,
None,
quant_config.w2_bias,
routing_data,
scatter_indx=scatter_idx,
gammas=None if apply_router_weight_on_input else gammas,
swizzle_mx_scale=None,
unpadded_N=unpadded_N_w2,
unpadded_K=unpadded_K_w2,
)
return out
class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
self.topk = moe_config.experts_per_token
self.renormalize = moe_config.routing_method in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
)
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
if not rocm_aiter_ops.is_enabled():
return False
from vllm.platforms.rocm import on_gfx950, on_gfx1250
return on_gfx950() or on_gfx1250()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) == (kMxfp4Static, None)
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU)
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return (
not moe_parallel_config.use_all2all_kernels
and not moe_parallel_config.enable_eplb
and moe_parallel_config.dp_size <= 1
)
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
@property
def expects_unquantized_inputs(self) -> bool:
return True
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
assert self.moe_config.intermediate_size_per_partition_unpadded is not None
assert self.moe_config.hidden_dim_unpadded is not None
score_mode = (
"sqrtsoftplus"
if self.moe_config.routing_method == RoutingMethodType.DeepseekV4
else None
)
return aiter_triton_kernel_w4a16_moe_forward(
hidden_states=hidden_states,
w1=w1,
w2=w2,
gating_output=router_logits,
topk=self.topk,
renormalize=self.renormalize,
activation=activation,
global_num_experts=global_num_experts,
expert_map=expert_map,
quant_config=self.quant_config,
apply_router_weight_on_input=apply_router_weight_on_input,
unpadded_N_w1=self.moe_config.intermediate_size_per_partition_unpadded * 2,
unpadded_K_w1=self.moe_config.hidden_dim_unpadded,
unpadded_N_w2=self.moe_config.hidden_dim_unpadded,
unpadded_K_w2=self.moe_config.intermediate_size_per_partition_unpadded,
num_expert_group=num_expert_group,
topk_group=topk_group,
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
score_mode=score_mode,
)
@@ -814,13 +814,13 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
) -> bool:
p = current_platform
if p.is_rocm():
from vllm.platforms.rocm import on_gfx9
from vllm.platforms.rocm import get_cdna_version
is_rocm_on_gfx9 = on_gfx9()
_rocm_support_fp8 = get_cdna_version() > 2
else:
is_rocm_on_gfx9 = False
_rocm_support_fp8 = False
device_supports_fp8 = is_rocm_on_gfx9 or (
device_supports_fp8 = _rocm_support_fp8 or (
p.is_cuda() and p.has_device_capability((8, 9))
)
@@ -472,11 +472,10 @@ class AiterExperts(mk.FusedMoEExpertsModular):
]
if (weight_key, activation_key) not in SUPPORTED_W_A:
return False
# CK MXFP4 MoE kernels are only supported on gfx950.
if weight_key == kMxfp4Static:
from vllm.platforms.rocm import on_gfx950
from vllm.platforms.rocm import on_gfx950, on_gfx1250
if not on_gfx950():
if not on_gfx950() or on_gfx1250():
return False
return True
@@ -222,11 +222,14 @@ def backend_to_kernel_cls(
return [BatchedMarlinExperts]
elif backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import (
AiterW4A16ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
)
return [AiterExperts]
return [AiterExperts, AiterW4A16ExpertsMonolithic]
elif backend == Mxfp4MoeBackend.AITER_MXFP4_FP8:
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import (
@@ -1263,6 +1266,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
Supports DeepGEMM, TRTLLM MXFP8, Triton and Marlin backends.
"""
from vllm.platforms.rocm import on_gfx1250
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
w13_weight_scale, w2_weight_scale = _pack_deepgemm_mxfp4_scales(
@@ -1435,7 +1439,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2_bias,
)
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and not on_gfx1250():
# Initially introduced for DeepSeekV4
if w13_bias is not None:
@@ -1492,7 +1496,9 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2_bias,
)
elif mxfp4_backend in TRITON_BACKENDS:
elif mxfp4_backend in TRITON_BACKENDS or (
mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and on_gfx1250()
):
from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig
if mxfp4_backend == Mxfp4MoeBackend.TRITON:
@@ -480,6 +480,10 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
@@ -735,7 +739,12 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
# For TRITON backends, weights are wrapped tensors from triton_kernels
# that don't support .detach(). Manually assign parameters.
if self.mxfp4_backend not in TRITON_BACKENDS:
from vllm.platforms.rocm import on_gfx1250
uses_triton_weight_format = self.mxfp4_backend in TRITON_BACKENDS or (
self.mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and on_gfx1250()
)
if not uses_triton_weight_format:
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
@@ -747,7 +756,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
self.w2_precision_config = w2_scale
# AITER backend requires weights to be marked as shuffled.
if self.mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
if (
self.mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16
and not uses_triton_weight_format
):
layer.w13_weight.is_shuffled = True
layer.w2_weight.is_shuffled = True
@@ -841,7 +853,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
w2_bias = getattr(layer, "w2_bias", None)
swiglu_limit = getattr(layer, "swiglu_limit", None)
if self.mxfp4_backend in TRITON_BACKENDS:
from vllm.platforms.rocm import on_gfx1250
if self.mxfp4_backend in TRITON_BACKENDS or (
self.mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and on_gfx1250()
):
# TRITON backends free w13/w2_weight_scale after swizzling; the
# swizzled scales live inside the precision configs instead.
assert self.w13_precision_config is not None
@@ -64,9 +64,7 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
kernel_supported_gpu = False
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx950
kernel_supported_gpu = on_gfx950()
kernel_supported_gpu = current_platform.supports_mx()
self.use_aiter_kernel = (
is_aiter_found_and_supported()
@@ -875,10 +875,40 @@ def w8a8_triton_block_scaled_mm(
Bs: The per-block quantization scale for `B`.
block_size: The block size for per-block quantization. It should
be 2-dim, e.g., [128, 128].
output_dtype: The dtype of the returned tensor.
output_dytpe: The dtype of the returned tensor.
Returns:
torch.Tensor: The result of matmul.
"""
from vllm.platforms.rocm import on_gfx1250
if on_gfx1250():
# Torch upcast reference: dequantize A,B to fp32 and matmul in fp32.
# Avoids the gfx1250 native-fp8 block GEMM NaN bug. Correct but slow.
_bn, _bk = block_size[0], block_size[1]
_As = (
_upcast_e8m0_to_fp32(As)
if As.dtype == torch.float8_e8m0fnu
else As.to(torch.float32)
)
_Bs = (
_upcast_e8m0_to_fp32(Bs)
if Bs.dtype == torch.float8_e8m0fnu
else Bs.to(torch.float32)
)
_K = A.shape[-1]
_N = B.shape[0]
_Af = A.to(torch.float32).reshape(-1, _K)
_Asf = (
_As.to(torch.float32)
.reshape(-1, _As.shape[-1])
.repeat_interleave(_bk, dim=1)[:, :_K]
)
_Bf = B.to(torch.float32)
_Bsf = _Bs.repeat_interleave(_bn, dim=0).repeat_interleave(_bk, dim=1)[:_N, :_K]
_out = (_Af * _Asf) @ (_Bf * _Bsf).t()
return _out.to(output_dtype).reshape(*A.shape[:-1], _N)
assert len(block_size) == 2
block_n, block_k = block_size[0], block_size[1]
+8 -2
View File
@@ -122,7 +122,7 @@ def use_aiter_triton_gemm(n, m, k, dtype):
def rocm_unquantized_gemm_impl(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx950
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx950, on_gfx1250
n = x.numel() // x.size(-1)
m = weight.shape[0]
@@ -164,7 +164,11 @@ def rocm_unquantized_gemm_impl(
if use_skinny_reduce_counting:
return ops.wvSplitKrc(x, weight, cu_count, bias)
if use_aiter_triton_gemm(n, m, k, x.dtype):
# gfx1250's aiter gemm_a16w16 uses the gluon backend, which requires
# K % 256 == 0 (it walks K with fixed-size descriptors and won't pad a
# partial last tile). Some whitelisted shapes have K=2880 (e.g. gpt-oss-120b
# hidden), so skip aiter there and fall back to the torch GEMM path below.
if use_aiter_triton_gemm(n, m, k, x.dtype) and not (on_gfx1250() and k % 256 != 0):
from aiter.ops.triton.gemm_a16w16 import gemm_a16w16
return gemm_a16w16(x, weight, bias)
@@ -172,6 +176,8 @@ def rocm_unquantized_gemm_impl(
use_skinny = (
envs.VLLM_ROCM_USE_SKINNY_GEMM
and (on_gfx9() or on_gfx1x())
# build (gfx9/gfx11 ISA); fall back to torch GEMM there.
# TODO GFX1250: Include once skinny GEMM is supported on gfx1250
and x.dtype in [torch.float16, torch.bfloat16]
and k % 8 == 0
)