Enable gfx1250 ROCm architecture (#46516)

Signed-off-by: Douglas Lehr <Doug.Lehr@amd.com>
Signed-off-by: jpvillam <juan.villamizar@amd.com>
Signed-off-by: Daniel <danichan@amd.com>
Signed-off-by: Jaden Mathias <jaden.mathias@amd.com>
Signed-off-by: jpvillam <jpvillam@amd.com>
Signed-off-by: Juan Villamizar <100237675+jpvillam-amd@users.noreply.github.com>
Signed-off-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Signed-off-by: Jaden Mathias <Jaden.Mathias@amd.com>
Co-authored-by: Douglas Lehr <Doug.Lehr@amd.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Co-authored-by: Daniel <danichan@amd.com>
Co-authored-by: Jaden Mathias <jaden.mathias@amd.com>
Co-authored-by: jpvillam <jpvillam@amd.com>
Co-authored-by: root <root@ctheliosr-1b114-f01-2.mnb.dcgpu>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: dllehr <douglehr@amd.com>
Co-authored-by: Daniel Chan <Daniel.Chan@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Juan Villamizar
2026-07-31 01:41:12 +00:00
committed by GitHub
co-authored by Douglas Lehr Claude Opus 4.7 Daniel Jaden Mathias jpvillam root Douglas Lehr dllehr Daniel Chan Andreas Karatzas
parent 553fcb82d5
commit 4f1da84eb5
33 changed files with 1850 additions and 104 deletions
+12 -7
View File
@@ -52,7 +52,7 @@ IS_AITER_FOUND = is_aiter_found()
def is_aiter_found_and_supported() -> bool:
"""Check if AITER library is available and platform supports it.
Checks: platform (ROCm), device arch (gfx9), and library existence.
Checks: platform (ROCm), device arch is CDNA 3 or better, and library existence.
Does NOT check environment variables - that's handled by rocm_aiter_ops.is_enabled().
This function determines if aiter CAN be used, not if it SHOULD be used.
@@ -66,9 +66,9 @@ def is_aiter_found_and_supported() -> bool:
VLLM_ROCM_USE_AITER=0, while preventing unwanted JIT warnings for auto-discovery.
"""
if current_platform.is_rocm() and IS_AITER_FOUND:
from vllm.platforms.rocm import on_mi3xx
from vllm.platforms.rocm import get_cdna_version
return on_mi3xx()
return get_cdna_version() > 2
return False
@@ -1798,16 +1798,21 @@ class rocm_aiter_ops:
@classmethod
@if_aiter_supported
def is_fp4bmm_enabled(cls) -> bool:
from vllm.platforms.rocm import on_gfx950
from vllm.platforms.rocm import get_cdna_version
return cls._AITER_ENABLED and cls._FP4BMM_ENABLED and on_gfx950()
# TODO GFX1250: Enable for cdna 4+ when aiter supports batched_gemm_a16wfp4 on gfx1250
return cls._AITER_ENABLED and cls._FP4BMM_ENABLED and get_cdna_version() == 4
@classmethod
@if_aiter_supported
def is_linear_hipbmm_enabled(cls) -> bool:
from vllm.platforms.rocm import on_mi3xx
from vllm.platforms.rocm import get_cdna_version
return cls.is_linear_enabled() and on_mi3xx() and cls._LINEAR_HIPBMM_ENABLED
return (
cls.is_linear_enabled()
and (get_cdna_version() > 2)
and cls._LINEAR_HIPBMM_ENABLED
)
@classmethod
@if_aiter_supported
@@ -374,9 +374,14 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
super().__init__(config)
n, k = config.weight_shape
self.use_triton = (
not current_platform.is_fp8_fnuz()
and rocm_aiter_ops.is_triton_gemm_w8a8_tuned(n, k)
_on_gfx1250 = False
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx1250
_on_gfx1250 = on_gfx1250()
self.use_triton = not current_platform.is_fp8_fnuz() and (
rocm_aiter_ops.is_triton_gemm_w8a8_tuned(n, k) or _on_gfx1250
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -104,10 +104,10 @@ class RowWiseTorchFP8ScaledMMLinearKernel(TorchFP8ScaledMMLinearKernel):
if not current_platform.is_rocm():
return False, "requires ROCm."
from vllm.platforms.rocm import on_mi3xx
from vllm.platforms.rocm import get_cdna_version
if not on_mi3xx():
return False, "requires MI3xx."
if get_cdna_version() <= 2:
return False, "requires CDNA3+"
if compute_capability is not None and compute_capability < 94:
return False, "requires compute capability 94 and above."
@@ -79,10 +79,15 @@ class ROCmFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
if not current_platform.is_rocm():
return False, "requires ROCm."
from vllm.platforms.rocm import on_gfx12x, on_mi3xx
from vllm.platforms.rocm import get_cdna_version, on_gfx12x, on_gfx1250
if not (on_mi3xx() or on_gfx12x()):
return False, "requires MI3xx or gfx12x"
# wvSplitKQ (skinny GEMM) is excluded from the gfx1250 build.
if on_gfx1250():
return False, "wvSplitKQ (skinny GEMM) is not built on gfx1250"
# Restore RDNA4 (gfx12x) dropped by the get_cdna_version()>2 refactor.
if get_cdna_version() <= 2 and not on_gfx12x():
return False, "requires CDNA3+ (gfx942/gfx950) or RDNA4 (gfx12x)"
if not envs.VLLM_ROCM_USE_SKINNY_GEMM:
return False, "requires VLLM_ROCM_USE_SKINNY_GEMM to be enabled."
@@ -20,7 +20,9 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
__all__ = [
"AiterW4A8ExpertsMonolithic",
"AiterW4A16ExpertsMonolithic",
"aiter_triton_kernel_w4a8_moe_forward",
"aiter_triton_kernel_w4a16_moe_forward",
]
@@ -46,11 +48,27 @@ def aiter_triton_kernel_w4a8_moe_forward(
and quant_config.use_mxfp4_w4a8
and rocm_aiter_ops.is_enabled()
)
from aiter.ops.triton.moe_routing.routing import routing as aiter_routing
from vllm.platforms.rocm import on_gfx1250
try:
from aiter.ops.triton.moe.moe_routing import routing as _routing_mod
except ImportError:
from aiter.ops.triton.moe_routing import routing as _routing_mod
if on_gfx1250():
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output, topk, sm_first=not renormalize
)
# gfx1250: aiter's in-kernel gather is numerically broken
if on_gfx1250():
gather_src = gather_idx.to(torch.long) // topk
hidden_states = hidden_states[gather_src]
gather_idx = None
return triton_kernel_fused_mxfp4_w4a8_experts(
None,
hidden_states,
@@ -199,12 +217,11 @@ class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
@staticmethod
def _supports_current_device() -> bool:
# Requires AITER and GFX950
if not rocm_aiter_ops.is_enabled():
return False
from vllm.platforms.rocm import on_gfx950
from vllm.platforms.rocm import on_gfx950, on_gfx1250
return on_gfx950()
return on_gfx950() or on_gfx1250()
@staticmethod
def _supports_no_act_and_mul() -> bool:
@@ -293,3 +310,358 @@ class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
unpadded_N_w2=self.moe_config.hidden_dim_unpadded,
unpadded_K_w2=self.moe_config.intermediate_size_per_partition_unpadded,
)
def _aiter_raw(t):
if t is None or isinstance(t, torch.Tensor):
return t
return t.storage.data if hasattr(t, "storage") else t
def _aiter_w4a16_silu_via_a8w4(
hidden_states: torch.Tensor,
w1_data,
w2_data,
w1_wscale,
w2_wscale,
w1_bias,
w2_bias,
routing_data,
gather_idx,
scatter_idx,
gammas,
apply_router_weight_on_input: bool,
swiglu_limit: float,
unpadded_N_w1,
unpadded_K_w1,
unpadded_N_w2,
unpadded_K_w2,
) -> torch.Tensor:
"""
MXFP4 w4a16 MoE with a SILU (concatenated ``[gate | up]``) activation.
"""
from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul
from aiter.ops.triton.moe_op_gemm_a8w4 import moe_gemm_a8w4
from aiter.ops.triton.quant import dynamic_mxfp8_quant
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
should_use_cdna4_mx_scale_swizzle,
)
swz = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None
quant_dtype = torch.float8_e4m3fn
g1_gammas = gammas if apply_router_weight_on_input else None
g2_gammas = None if apply_router_weight_on_input else gammas
hidden_q, a1_scale = dynamic_mxfp8_quant(hidden_states, quant_dtype=quant_dtype)
raw_gate_up = moe_gemm_a8w4(
hidden_q,
w1_data,
a1_scale,
w1_wscale,
None,
None,
w1_bias,
routing_data,
gather_indx=gather_idx,
gammas=g1_gammas,
swizzle_mx_scale=swz,
out_dtype=torch.bfloat16,
apply_swiglu=False,
unpadded_N=unpadded_N_w1,
unpadded_K=unpadded_K_w1,
)
if unpadded_N_w1 is not None:
raw_gate_up = raw_gate_up[:, :unpadded_N_w1]
interim_fp8, a2_scale = fused_clamp_act_mul(
raw_gate_up,
swiglu_limit=swiglu_limit,
activation="silu",
dtype_quant=quant_dtype,
scale_dtype_fmt="ue8m0",
quant_block_size=32,
)
out = moe_gemm_a8w4(
interim_fp8,
w2_data,
a2_scale,
w2_wscale,
None,
None,
w2_bias,
routing_data,
scatter_indx=scatter_idx,
gammas=g2_gammas,
swizzle_mx_scale=swz,
unpadded_N=unpadded_N_w2,
unpadded_K=unpadded_K_w2,
)
return out
def aiter_triton_kernel_w4a16_moe_forward(
hidden_states: torch.Tensor,
w1,
w2,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
activation: MoEActivation = MoEActivation.SWIGLUOAI,
quant_config: FusedMoEQuantConfig | None = None,
apply_router_weight_on_input: bool = False,
global_num_experts: int = -1,
expert_map: torch.Tensor | None = None,
unpadded_N_w1=None,
unpadded_K_w1=None,
unpadded_N_w2=None,
unpadded_K_w2=None,
num_expert_group: int | None = None,
topk_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
score_mode: str | None = None,
):
assert quant_config is not None and rocm_aiter_ops.is_enabled()
from vllm.platforms.rocm import on_gfx1250
try:
from aiter.ops.triton.moe.moe_op_gemm_a16w4 import moe_gemm_a16w4
from aiter.ops.triton.moe.moe_routing import routing as _routing_mod
except ImportError:
from aiter.ops.triton.moe.moe_op_gemm_a16w4 import moe_gemm_a16w4
from aiter.ops.triton.moe_routing import routing as _routing_mod
if on_gfx1250():
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
if score_mode is not None:
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output,
topk,
score_mode=score_mode,
bias=e_score_correction_bias,
renorm=renormalize,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
)
else:
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output, topk, sm_first=not renormalize
)
if on_gfx1250():
gather_src = gather_idx.to(torch.long) // topk
hidden_states = hidden_states[gather_src]
gather_idx = None
assert quant_config.w1_precision is not None
assert quant_config.w2_precision is not None
w1_data = _aiter_raw(w1)
w2_data = _aiter_raw(w2)
w1_wscale = _aiter_raw(quant_config.w1_precision.weight_scale)
w2_wscale = _aiter_raw(quant_config.w2_precision.weight_scale)
gammas = routing_data.gate_scal if routing_data else None
swiglu_alpha = (
quant_config.gemm1_alpha if quant_config.gemm1_alpha is not None else 1.0
)
swiglu_limit = (
quant_config.gemm1_clamp_limit
if quant_config.gemm1_clamp_limit is not None
else 7.0
)
# SILU on gfx1250: use the verified a8w4 kernel (dynamic MXFP8); a16w4 faults.
if activation == MoEActivation.SILU and on_gfx1250():
return _aiter_w4a16_silu_via_a8w4(
hidden_states,
w1_data,
w2_data,
w1_wscale,
w2_wscale,
quant_config.w1_bias,
quant_config.w2_bias,
routing_data,
gather_idx,
scatter_idx,
gammas,
apply_router_weight_on_input,
swiglu_limit,
unpadded_N_w1,
unpadded_K_w1,
unpadded_N_w2,
unpadded_K_w2,
)
# SILU: silu(gate) * up — same kernel, just no "+1" residual in swiglu.
swiglu_add_residual = activation != MoEActivation.SILU
intermediate = moe_gemm_a16w4(
hidden_states,
w1_data,
None,
w1_wscale,
None,
None,
quant_config.w1_bias,
routing_data,
gather_indx=gather_idx,
gammas=gammas if apply_router_weight_on_input else None,
swizzle_mx_scale=None,
apply_swiglu=True,
alpha=swiglu_alpha,
limit=swiglu_limit,
swiglu_add_residual=swiglu_add_residual,
unpadded_N=unpadded_N_w1,
unpadded_K=unpadded_K_w1,
)
out = moe_gemm_a16w4(
intermediate,
w2_data,
None,
w2_wscale,
None,
None,
quant_config.w2_bias,
routing_data,
scatter_indx=scatter_idx,
gammas=None if apply_router_weight_on_input else gammas,
swizzle_mx_scale=None,
unpadded_N=unpadded_N_w2,
unpadded_K=unpadded_K_w2,
)
return out
class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
self.topk = moe_config.experts_per_token
self.renormalize = moe_config.routing_method in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
)
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
if not rocm_aiter_ops.is_enabled():
return False
from vllm.platforms.rocm import on_gfx950, on_gfx1250
return on_gfx950() or on_gfx1250()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) == (kMxfp4Static, None)
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU)
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return (
not moe_parallel_config.use_all2all_kernels
and not moe_parallel_config.enable_eplb
and moe_parallel_config.dp_size <= 1
)
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
@property
def expects_unquantized_inputs(self) -> bool:
return True
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
assert self.moe_config.intermediate_size_per_partition_unpadded is not None
assert self.moe_config.hidden_dim_unpadded is not None
score_mode = (
"sqrtsoftplus"
if self.moe_config.routing_method == RoutingMethodType.DeepseekV4
else None
)
return aiter_triton_kernel_w4a16_moe_forward(
hidden_states=hidden_states,
w1=w1,
w2=w2,
gating_output=router_logits,
topk=self.topk,
renormalize=self.renormalize,
activation=activation,
global_num_experts=global_num_experts,
expert_map=expert_map,
quant_config=self.quant_config,
apply_router_weight_on_input=apply_router_weight_on_input,
unpadded_N_w1=self.moe_config.intermediate_size_per_partition_unpadded * 2,
unpadded_K_w1=self.moe_config.hidden_dim_unpadded,
unpadded_N_w2=self.moe_config.hidden_dim_unpadded,
unpadded_K_w2=self.moe_config.intermediate_size_per_partition_unpadded,
num_expert_group=num_expert_group,
topk_group=topk_group,
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
score_mode=score_mode,
)
@@ -814,13 +814,13 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
) -> bool:
p = current_platform
if p.is_rocm():
from vllm.platforms.rocm import on_gfx9
from vllm.platforms.rocm import get_cdna_version
is_rocm_on_gfx9 = on_gfx9()
_rocm_support_fp8 = get_cdna_version() > 2
else:
is_rocm_on_gfx9 = False
_rocm_support_fp8 = False
device_supports_fp8 = is_rocm_on_gfx9 or (
device_supports_fp8 = _rocm_support_fp8 or (
p.is_cuda() and p.has_device_capability((8, 9))
)
@@ -472,11 +472,10 @@ class AiterExperts(mk.FusedMoEExpertsModular):
]
if (weight_key, activation_key) not in SUPPORTED_W_A:
return False
# CK MXFP4 MoE kernels are only supported on gfx950.
if weight_key == kMxfp4Static:
from vllm.platforms.rocm import on_gfx950
from vllm.platforms.rocm import on_gfx950, on_gfx1250
if not on_gfx950():
if not on_gfx950() or on_gfx1250():
return False
return True
@@ -222,11 +222,14 @@ def backend_to_kernel_cls(
return [BatchedMarlinExperts]
elif backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import (
AiterW4A16ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
)
return [AiterExperts]
return [AiterExperts, AiterW4A16ExpertsMonolithic]
elif backend == Mxfp4MoeBackend.AITER_MXFP4_FP8:
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import (
@@ -1263,6 +1266,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
Supports DeepGEMM, TRTLLM MXFP8, Triton and Marlin backends.
"""
from vllm.platforms.rocm import on_gfx1250
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
w13_weight_scale, w2_weight_scale = _pack_deepgemm_mxfp4_scales(
@@ -1435,7 +1439,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2_bias,
)
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and not on_gfx1250():
# Initially introduced for DeepSeekV4
if w13_bias is not None:
@@ -1492,7 +1496,9 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2_bias,
)
elif mxfp4_backend in TRITON_BACKENDS:
elif mxfp4_backend in TRITON_BACKENDS or (
mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and on_gfx1250()
):
from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig
if mxfp4_backend == Mxfp4MoeBackend.TRITON:
@@ -480,6 +480,10 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
@@ -735,7 +739,12 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
# For TRITON backends, weights are wrapped tensors from triton_kernels
# that don't support .detach(). Manually assign parameters.
if self.mxfp4_backend not in TRITON_BACKENDS:
from vllm.platforms.rocm import on_gfx1250
uses_triton_weight_format = self.mxfp4_backend in TRITON_BACKENDS or (
self.mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and on_gfx1250()
)
if not uses_triton_weight_format:
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
@@ -747,7 +756,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
self.w2_precision_config = w2_scale
# AITER backend requires weights to be marked as shuffled.
if self.mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
if (
self.mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16
and not uses_triton_weight_format
):
layer.w13_weight.is_shuffled = True
layer.w2_weight.is_shuffled = True
@@ -841,7 +853,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
w2_bias = getattr(layer, "w2_bias", None)
swiglu_limit = getattr(layer, "swiglu_limit", None)
if self.mxfp4_backend in TRITON_BACKENDS:
from vllm.platforms.rocm import on_gfx1250
if self.mxfp4_backend in TRITON_BACKENDS or (
self.mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and on_gfx1250()
):
# TRITON backends free w13/w2_weight_scale after swizzling; the
# swizzled scales live inside the precision configs instead.
assert self.w13_precision_config is not None
@@ -64,9 +64,7 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
kernel_supported_gpu = False
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx950
kernel_supported_gpu = on_gfx950()
kernel_supported_gpu = current_platform.supports_mx()
self.use_aiter_kernel = (
is_aiter_found_and_supported()
@@ -875,10 +875,40 @@ def w8a8_triton_block_scaled_mm(
Bs: The per-block quantization scale for `B`.
block_size: The block size for per-block quantization. It should
be 2-dim, e.g., [128, 128].
output_dtype: The dtype of the returned tensor.
output_dytpe: The dtype of the returned tensor.
Returns:
torch.Tensor: The result of matmul.
"""
from vllm.platforms.rocm import on_gfx1250
if on_gfx1250():
# Torch upcast reference: dequantize A,B to fp32 and matmul in fp32.
# Avoids the gfx1250 native-fp8 block GEMM NaN bug. Correct but slow.
_bn, _bk = block_size[0], block_size[1]
_As = (
_upcast_e8m0_to_fp32(As)
if As.dtype == torch.float8_e8m0fnu
else As.to(torch.float32)
)
_Bs = (
_upcast_e8m0_to_fp32(Bs)
if Bs.dtype == torch.float8_e8m0fnu
else Bs.to(torch.float32)
)
_K = A.shape[-1]
_N = B.shape[0]
_Af = A.to(torch.float32).reshape(-1, _K)
_Asf = (
_As.to(torch.float32)
.reshape(-1, _As.shape[-1])
.repeat_interleave(_bk, dim=1)[:, :_K]
)
_Bf = B.to(torch.float32)
_Bsf = _Bs.repeat_interleave(_bn, dim=0).repeat_interleave(_bk, dim=1)[:_N, :_K]
_out = (_Af * _Asf) @ (_Bf * _Bsf).t()
return _out.to(output_dtype).reshape(*A.shape[:-1], _N)
assert len(block_size) == 2
block_n, block_k = block_size[0], block_size[1]
+8 -2
View File
@@ -122,7 +122,7 @@ def use_aiter_triton_gemm(n, m, k, dtype):
def rocm_unquantized_gemm_impl(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx950
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx950, on_gfx1250
n = x.numel() // x.size(-1)
m = weight.shape[0]
@@ -164,7 +164,11 @@ def rocm_unquantized_gemm_impl(
if use_skinny_reduce_counting:
return ops.wvSplitKrc(x, weight, cu_count, bias)
if use_aiter_triton_gemm(n, m, k, x.dtype):
# gfx1250's aiter gemm_a16w16 uses the gluon backend, which requires
# K % 256 == 0 (it walks K with fixed-size descriptors and won't pad a
# partial last tile). Some whitelisted shapes have K=2880 (e.g. gpt-oss-120b
# hidden), so skip aiter there and fall back to the torch GEMM path below.
if use_aiter_triton_gemm(n, m, k, x.dtype) and not (on_gfx1250() and k % 256 != 0):
from aiter.ops.triton.gemm_a16w16 import gemm_a16w16
return gemm_a16w16(x, weight, bias)
@@ -172,6 +176,8 @@ def rocm_unquantized_gemm_impl(
use_skinny = (
envs.VLLM_ROCM_USE_SKINNY_GEMM
and (on_gfx9() or on_gfx1x())
# build (gfx9/gfx11 ISA); fall back to torch GEMM there.
# TODO GFX1250: Include once skinny GEMM is supported on gfx1250
and x.dtype in [torch.float16, torch.bfloat16]
and k % 8 == 0
)
+37 -13
View File
@@ -194,11 +194,6 @@ def _get_gcn_arch() -> str:
return _query_gcn_arch_from_amdsmi()
except Exception as e:
logger.debug("Failed to get GCN arch via amdsmi: %s", e)
logger.warning_once(
"Failed to get GCN arch via amdsmi, falling back to torch.cuda. "
"This will initialize CUDA and may cause "
"issues if CUDA_VISIBLE_DEVICES is not set yet."
)
# Ultimate fallback: use torch.cuda (will initialize CUDA)
return torch.cuda.get_device_properties("cuda").gcnArchName
@@ -218,6 +213,11 @@ _ON_GFX9 = any(arch in _GCN_ARCH for arch in ["gfx90a", "gfx942", "gfx950"])
_ON_GFX90A = "gfx90a" in _GCN_ARCH
_ON_GFX942 = "gfx942" in _GCN_ARCH
_ON_GFX950 = "gfx950" in _GCN_ARCH
_ON_GFX1250 = "gfx1250" in _GCN_ARCH
_ON_CDNA = any(arch in _GCN_ARCH for arch in ["gfx9", "gfx1250"])
# RDNA = gfx11/gfx12 minus the CDNA-classified gfx1250.
_ON_RDNA = _ON_GFX1X and not _ON_CDNA
def _capability_from_gcn_arch(gcn_arch: str) -> tuple[int, int] | None:
@@ -292,7 +292,7 @@ def _capability_from_gcn_arch(gcn_arch: str) -> tuple[int, int] | None:
def on_gfx1x() -> bool:
return _ON_GFX1X
return _ON_GFX1X and not _ON_CDNA
def on_gfx11() -> bool:
@@ -308,7 +308,11 @@ def on_gfx1151() -> bool:
def on_gfx12x() -> bool:
return _ON_GFX12X
return _ON_GFX12X and not _ON_CDNA
def on_gfx1250() -> bool:
return _ON_GFX1250
def on_mi3xx() -> bool:
@@ -331,13 +335,33 @@ def on_gfx950() -> bool:
return _ON_GFX950
def on_cdna() -> bool:
return _ON_CDNA
def on_rdna() -> bool:
return _ON_RDNA
def get_cdna_version() -> int:
if on_gfx90a():
return 2
if on_gfx942():
return 3
if on_gfx950():
return 4
if on_gfx1250():
return 5
return 0
# Enable HIP online tuning early, before hipBLASLt initializes.
# Turn on hipBLASLt online tuning if use AITER hipBLASLt GEMM.
if (
envs.VLLM_ROCM_USE_AITER
and envs.VLLM_ROCM_USE_AITER_LINEAR
and envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM
and on_mi3xx()
and get_cdna_version() > 2
):
os.environ["HIP_ONLINE_TUNING"] = "1"
@@ -356,7 +380,7 @@ def use_rocm_custom_paged_attention(
) -> bool:
# custom paged attn always supported on V0. On V1, requires sliding window
# disabled due to observed numerical discrepancy.
if _ON_GFX9:
if on_cdna():
return (
(sliding_window == 0 or sliding_window == (-1, -1))
and (qtype == torch.half or qtype == torch.bfloat16)
@@ -657,12 +681,12 @@ class RocmPlatform(Platform):
from vllm._aiter_ops import rocm_aiter_ops
if rocm_aiter_ops.is_mha_enabled() and on_gfx9():
if rocm_aiter_ops.is_mha_enabled() and on_cdna():
logger.info_once("Using AITER Flash Attention backend for ViT model.")
return AttentionBackendEnum.ROCM_AITER_FA
if (
on_gfx9()
on_cdna()
and find_spec("flash_attn") is not None
and (dtype == torch.float16 or dtype == torch.bfloat16)
):
@@ -878,11 +902,11 @@ class RocmPlatform(Platform):
@classmethod
def supports_mx(cls) -> bool:
return any(gfx in _GCN_ARCH for gfx in ["gfx95"])
return any(gfx in _GCN_ARCH for gfx in ["gfx95", "gfx1250"])
@classmethod
def supports_fp8(cls) -> bool:
return on_gfx9() or on_gfx12x()
return on_cdna() or on_gfx12x()
@classmethod
def is_fp8_fnuz(cls) -> bool:
+12 -1
View File
@@ -505,7 +505,18 @@ def has_triton_kernels() -> bool:
@cache
def has_tilelang() -> bool:
"""Whether the optional `tilelang` package is available."""
return _has_module("tilelang")
if not _has_module("tilelang"):
return False
# ROCm-only guard, imported lazily to avoid loading rocm on CUDA.
from vllm.platforms import current_platform
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx1250
# TODO: Re-enable when tilelang supports gfx1250
if on_gfx1250():
return False
return True
def has_arctic_inference() -> bool:
+18 -2
View File
@@ -231,10 +231,26 @@ def _log_triton_jit_compile(fn_name: str, kwargs) -> None:
def _setup_triton_jit_hook() -> None:
"""Register a ``jit_post_compile_hook`` that warns on compilation."""
"""Register a jit_post_compile_hook that warns on compilation."""
if not HAS_TRITON:
return
from triton import knobs # type: ignore[import-untyped]
from triton import knobs
from triton.runtime import jit as _triton_jit
# kernels pass non-JSON-serializable constexprs
# make that serialization non-fatal
_orig = _triton_jit.serialize_specialization_data
if not getattr(_orig, "_vllm_guarded", False):
@functools.wraps(_orig)
def _guarded(*args, **kwargs):
try:
return _orig(*args, **kwargs)
except (TypeError, ValueError):
return None # best-effort metadata; monitor ignores it
cast(Any, _guarded)._vllm_guarded = True
_triton_jit.serialize_specialization_data = _guarded
existing_hook = knobs.runtime.jit_post_compile_hook
+18 -9
View File
@@ -35,23 +35,31 @@ elif current_platform.is_xpu():
compile_flash_attn_varlen_func_from_specs = None # type: ignore[assignment]
get_scheduler_metadata = xpu_ops.get_scheduler_metadata # type: ignore[assignment]
elif current_platform.is_rocm():
# On ROCm we use AITER's Triton flash-attention; the upstream flash-attn
# package is not installed/available. (Same source as aiter_triton_mla.py.)
# The FA4 compile-from-specs API is CUDA-only, so it is unavailable on ROCm
# regardless of whether AITER is present.
from vllm.platforms.rocm import on_gfx1250
compile_flash_attn_varlen_func_from_specs = None # type: ignore[assignment]
try:
from flash_attn import flash_attn_varlen_func # type: ignore[no-redef]
if on_gfx1250():
from aiter.ops.triton.mha import ( # type: ignore[no-redef]
flash_attn_varlen_func,
)
else:
from flash_attn import flash_attn_varlen_func # type: ignore[no-redef]
compile_flash_attn_varlen_func_from_specs = None # type: ignore[assignment]
# Mark that upstream flash-attn is available on ROCm
_ROCM_FLASH_ATTN_AVAILABLE = True
except ImportError:
def flash_attn_varlen_func(*args: Any, **kwargs: Any) -> Any: # type: ignore[no-redef,misc]
package = "aiter" if on_gfx1250() else "flash-attn"
raise ImportError(
"ROCm platform requires upstream flash-attn "
"to be installed. Please install flash-attn first."
f"ROCm platform requires upstream {package} "
f"to be installed. Please install {package} first."
)
compile_flash_attn_varlen_func_from_specs = None # type: ignore[assignment]
# ROCm doesn't use scheduler metadata (FA3 feature), provide stub
def get_scheduler_metadata(*args: Any, **kwargs: Any) -> None: # type: ignore[misc]
return None
@@ -348,7 +356,8 @@ def is_flash_attn_varlen_func_available() -> bool:
Platform-specific sources:
- CUDA: vllm.vllm_flash_attn.flash_attn_varlen_func
- XPU: xpu_ops.flash_attn_varlen_func
- ROCm: upstream flash_attn.flash_attn_varlen_func (if available)
- ROCm: aiter.ops.triton.mha.flash_attn_varlen_func (if AITER available) or
upstream flash_attn.flash_attn_varlen_func
Note: This is separate from the AITER flash attention backend (rocm_aiter_fa.py)
which uses rocm_aiter_ops.flash_attn_varlen_func. The condition to use AITER is
+3 -3
View File
@@ -776,12 +776,12 @@ class AiterFlashAttentionBackend(AttentionBackend):
@classmethod
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
from vllm.platforms.rocm import on_mi3xx
from vllm.platforms.rocm import get_cdna_version
# DeviceCapability is currently created using torch.cuda.get_device_capability()
# which is known to be buggy on rocm systems. on_mi3xx uses amd-smi which is
# which is known to be buggy on rocm systems. on CDNA uses amd-smi which is
# more reliable.
return on_mi3xx()
return get_cdna_version() > 2
@classmethod
def supports_non_causal(cls) -> bool:
+8
View File
@@ -18,6 +18,13 @@ if HAS_TRITON:
logger = init_logger(__name__)
def _skip_aiter_sampler_on_gfx1250() -> bool:
# Lazy ROCm-only import; keeps arch detection out of import time on CUDA/CPU.
from vllm.platforms.rocm import on_gfx1250
return on_gfx1250()
def flashinfer_sampler_supported() -> bool:
"""Decide whether FlashInfer's top-p/top-k sampler can be used.
@@ -110,6 +117,7 @@ class TopKTopPSampler(nn.Module):
elif (
logprobs_mode not in PROCESSED_LOGPROBS_MODES
and rocm_aiter_ops.is_enabled()
and not _skip_aiter_sampler_on_gfx1250() # TODO (JPVILLAM): Enable
):
self.aiter_ops = None
self._aiter_ops_import_failed = False