[ROCm] triton+triton_kernels 3.8 mxfp4 MoE support (gpt-oss + DeepSeek-V4) (#55934)
Signed-off-by: Rohan Potdar <rohan.potdar@amd.com> Signed-off-by: Rohan138 <rohanpotdar138@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Matt <156021403+mawong-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Matt
parent
95f4925c3a
commit
3e267bae70
@@ -7,7 +7,7 @@ import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.import_utils import has_triton_kernels
|
||||
from vllm.utils.import_utils import get_triton_kernels_version, has_triton_kernels
|
||||
from vllm.utils.torch_utils import direct_register_custom_op, is_torch_equal_or_newer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -27,10 +27,23 @@ def should_use_cdna4_mx_scale_swizzle() -> bool:
|
||||
return on_gfx950() and get_tensor_model_parallel_world_size() <= 2
|
||||
|
||||
|
||||
def weight_mx_scale(precision_config):
|
||||
"""mxfp4 weight scale from a triton_kernels PrecisionConfig: 3.8 exposes it
|
||||
as b_mx_scale, 3.5.1/3.6 as weight_scale.
|
||||
"""
|
||||
if get_triton_kernels_version() == "3.8":
|
||||
return precision_config.b_mx_scale
|
||||
return precision_config.weight_scale
|
||||
|
||||
|
||||
def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
|
||||
"""Weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel."""
|
||||
assert has_triton_kernels()
|
||||
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
|
||||
if get_triton_kernels_version() == "3.8":
|
||||
# 3.8: matmul_ogs_details -> matmul_details
|
||||
import triton_kernels.matmul_details.opt_flags as opt_flags
|
||||
else:
|
||||
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
|
||||
from triton_kernels.numerics import InFlexData
|
||||
from triton_kernels.tensor import FP4, convert_layout, wrap_torch_tensor
|
||||
from triton_kernels.tensor_details import layout
|
||||
@@ -51,6 +64,17 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
|
||||
)
|
||||
value_layout = StridedLayout
|
||||
scale_layout = StridedLayout
|
||||
elif current_platform.is_rocm() and get_triton_kernels_version() == "3.8":
|
||||
# StridedLayout(-2) is triton_kernels' default CDNA mxfp4 weight layout
|
||||
# (make_default_matmul_mxfp4_w_layout); 3.8 wants layout instances.
|
||||
from triton_kernels.tensor_details.layout import CDNA4MXScaleLayout
|
||||
|
||||
value_layout = StridedLayout(-2)
|
||||
scale_layout = (
|
||||
CDNA4MXScaleLayout()
|
||||
if should_use_cdna4_mx_scale_swizzle()
|
||||
else StridedLayout()
|
||||
)
|
||||
elif current_platform.is_rocm():
|
||||
value_layout = StridedLayout
|
||||
if should_use_cdna4_mx_scale_swizzle():
|
||||
|
||||
Reference in New Issue
Block a user