[ROCm] gfx1250 optimizations and model enablement (#58800)
Signed-off-by: jpvillam <juan.villamizar@amd.com> Signed-off-by: Jaden Mathias <jaden.mathias@amd.com> Signed-off-by: Juan Villamizar <jpvillam@amd.com> Signed-off-by: jpvillam <jpvillam@amd.com> Signed-off-by: Omar Muhammad <omar.muhammad@amd.com> Signed-off-by: Daniel <danichan@amd.com> Signed-off-by: Juan Villamizar <100237675+jpvillam-amd@users.noreply.github.com> Co-authored-by: Jaden Mathias <jaden.mathias@amd.com> Co-authored-by: jpvillam <jpvillam@amd.com> Co-authored-by: Omar Muhammad <omar.muhammad@amd.com> Co-authored-by: Daniel <danichan@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Daniel Chan <Daniel.Chan@amd.com>
This commit is contained in:
co-authored by
Jaden Mathias
jpvillam
Omar Muhammad
Daniel
Claude Opus 5
Daniel Chan
parent
9b99ccb56b
commit
e711723352
@@ -13,6 +13,14 @@ from vllm.utils.torch_utils import direct_register_custom_op, is_torch_equal_or_
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def should_use_gfx1250_mx_scale_swizzle() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx1250
|
||||
|
||||
return on_gfx1250()
|
||||
|
||||
|
||||
def should_use_cdna4_mx_scale_swizzle() -> bool:
|
||||
"""Whether to use the CDNA4 swizzled scale layout for mxfp4 on gfx950.
|
||||
|
||||
@@ -27,6 +35,15 @@ def should_use_cdna4_mx_scale_swizzle() -> bool:
|
||||
return on_gfx950() and get_tensor_model_parallel_world_size() <= 2
|
||||
|
||||
|
||||
def get_aiter_mx_scale_swizzle() -> str | None:
|
||||
"""AITER `swizzle_mx_scale` arg matching the load-time scale swizzle."""
|
||||
if should_use_gfx1250_mx_scale_swizzle():
|
||||
return "GFX1250_SCALE"
|
||||
if should_use_cdna4_mx_scale_swizzle():
|
||||
return "CDNA4_SCALE"
|
||||
return None
|
||||
|
||||
|
||||
def weight_mx_scale(precision_config):
|
||||
"""mxfp4 weight scale from a triton_kernels PrecisionConfig: 3.8 exposes it
|
||||
as b_mx_scale, 3.5.1/3.6 as weight_scale.
|
||||
@@ -136,6 +153,17 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
|
||||
# transpose the tensor so that the quantization axis is on dim1
|
||||
quant_tensor = quant_tensor.transpose(-2, -1)
|
||||
scale = scale.transpose(-2, -1)
|
||||
|
||||
if should_use_gfx1250_mx_scale_swizzle():
|
||||
from aiter.ops.triton.utils.shuffle import shuffle_scale_moe
|
||||
|
||||
assert (
|
||||
scale.dim() == 3 and scale.shape[-1] % 32 == 0 and scale.shape[-2] % 4 == 0
|
||||
), f"GFX1250 scale swizzle needs (E,K_SCALE%4,N%32); got {tuple(scale.shape)}"
|
||||
scale = shuffle_scale_moe(
|
||||
scale, arch="gfx1250", preshuffle_factor=32, scale_kwidth=4
|
||||
)
|
||||
|
||||
quant_tensor = convert_layout(
|
||||
wrap_torch_tensor(quant_tensor, dtype=FP4), value_layout, **value_layout_opts
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user