[ROCm][Quantization][2/N] Refactor quark_moe w4a8 w/ oracle (#39136)
Signed-off-by: Bowen Bao <bowenbao@amd.com>
This commit is contained in:
@@ -28,6 +28,25 @@ HOPPER_MXFP4_BF16_AVAILABLE = (
|
||||
and has_flashinfer()
|
||||
)
|
||||
|
||||
# ROCm platform and dependencies
|
||||
ROCM_AVAILABLE = current_platform.is_rocm()
|
||||
ROCM_TRITON_KERNELS_AVAILABLE = False
|
||||
ROCM_AITER_AVAILABLE = False
|
||||
ROCM_GFX950 = False
|
||||
|
||||
if ROCM_AVAILABLE:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.utils.import_utils import has_triton_kernels
|
||||
|
||||
ROCM_TRITON_KERNELS_AVAILABLE = has_triton_kernels()
|
||||
ROCM_GFX950 = on_gfx950()
|
||||
ROCM_AITER_AVAILABLE = rocm_aiter_ops.is_enabled()
|
||||
|
||||
if ROCM_AITER_AVAILABLE:
|
||||
from aiter.ops.triton.moe.quant_moe import upcast_from_mxfp
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
if TRTLLM_GEN_MXFP4_AVAILABLE:
|
||||
from flashinfer import (
|
||||
fp4_quantize,
|
||||
@@ -111,6 +130,7 @@ def test_mxfp4_loading_and_execution_moe(vllm_runner, model_case: ModelCase):
|
||||
|
||||
def swiglu(x, alpha: float = 1.702, beta: float = 1.0, limit: float | None = None):
|
||||
# Note we add an extra bias of 1 to the linear layer
|
||||
# Uses chunked layout: first half is gate, second half is up
|
||||
x_glu, x_linear = torch.chunk(x, 2, dim=-1)
|
||||
if limit is not None:
|
||||
x_glu = x_glu.clamp(max=limit)
|
||||
@@ -119,6 +139,16 @@ def swiglu(x, alpha: float = 1.702, beta: float = 1.0, limit: float | None = Non
|
||||
return out_glu * (x_linear + beta)
|
||||
|
||||
|
||||
def swigluoai(x, alpha: float = 1.702, limit: float = 7.0):
|
||||
# OAI swiglu uses interleaved layout: gate/up alternating
|
||||
# See SwigluOAIAndMul in vllm/model_executor/layers/activation.py
|
||||
gate, up = x[..., ::2], x[..., 1::2]
|
||||
gate = gate.clamp(max=limit)
|
||||
up = up.clamp(min=-limit, max=limit)
|
||||
glu = gate * torch.sigmoid(gate * alpha)
|
||||
return (up + 1) * glu
|
||||
|
||||
|
||||
fp4_lookup_table = [0, 0.5, 1, 1.5, 2, 3, 4, 6, -0, -0.5, -1, -1.5, -2, -3, -4, -6]
|
||||
|
||||
|
||||
@@ -168,8 +198,20 @@ def reference_moe(
|
||||
beta,
|
||||
limit,
|
||||
act_type,
|
||||
is_gated,
|
||||
activation: str = "swiglu",
|
||||
use_interleaved_layout: bool = False,
|
||||
):
|
||||
"""
|
||||
Reference MoE implementation for accuracy testing.
|
||||
|
||||
Args:
|
||||
activation: One of "swiglu", "silu", "relu2". Controls the activation
|
||||
function used after the first MLP.
|
||||
use_interleaved_layout: If True, uses interleaved gate/up layout
|
||||
(gate=x[..., ::2], up=x[..., 1::2]) as used by SWIGLUOAI.
|
||||
If False, uses chunked layout (gate, up = chunk(x, 2)) as used
|
||||
by standard swiglu/silu.
|
||||
"""
|
||||
# renormalize routing
|
||||
experts = torch.topk(roouting_logits, k=topk, dim=-1, sorted=True)
|
||||
expert_weights = torch.nn.functional.softmax(experts.values, dim=1)
|
||||
@@ -179,12 +221,21 @@ def reference_moe(
|
||||
mlp1_weight = w13[expert_indices, ...]
|
||||
mlp1_bias = bias13[expert_indices, ...]
|
||||
t = torch.einsum("beck,bk->bec", mlp1_weight, t) + mlp1_bias
|
||||
if is_gated:
|
||||
t = swiglu(t, alpha=alpha, beta=beta, limit=limit)
|
||||
else:
|
||||
|
||||
# Apply activation
|
||||
if activation in ("swiglu", "silu"):
|
||||
if use_interleaved_layout:
|
||||
# SWIGLUOAI: interleaved gate/up layout
|
||||
t = swigluoai(t, alpha=alpha, limit=limit)
|
||||
else:
|
||||
# Standard swiglu/silu: chunked layout
|
||||
t = swiglu(t, alpha=alpha, beta=beta, limit=limit)
|
||||
elif activation == "relu2":
|
||||
# RELU2_NO_MUL: relu(x)^2
|
||||
t = torch.relu(t)
|
||||
t = t * t
|
||||
else:
|
||||
raise ValueError(f"Unknown activation: {activation}")
|
||||
|
||||
if act_type == "mxfp8":
|
||||
t_quantized, t_scale = mxfp8_quantize(
|
||||
@@ -585,7 +636,8 @@ def test_trtllm_gen_mxfp4_fused_moe(
|
||||
beta,
|
||||
limit,
|
||||
act_type,
|
||||
is_gated=True,
|
||||
activation="swiglu",
|
||||
use_interleaved_layout=False,
|
||||
)
|
||||
ref_result[start_idx:end_idx].copy_(chunk_result)
|
||||
|
||||
@@ -722,7 +774,8 @@ def test_flashinfer_cutlass_mxfp4_fused_moe(
|
||||
beta,
|
||||
limit,
|
||||
"bf16",
|
||||
is_gated=True,
|
||||
activation="swiglu",
|
||||
use_interleaved_layout=False,
|
||||
)
|
||||
|
||||
from vllm.utils.flashinfer import flashinfer_cutlass_fused_moe
|
||||
@@ -908,7 +961,8 @@ def test_flashinfer_cutlass_mxfp4_mxfp8_fused_moe(
|
||||
beta,
|
||||
limit,
|
||||
"mxfp8",
|
||||
is_gated=True,
|
||||
activation="swiglu",
|
||||
use_interleaved_layout=False,
|
||||
)
|
||||
|
||||
# Prepare inputs for FlashInfer CUTLASS fused MoE
|
||||
@@ -1080,7 +1134,8 @@ def test_trtllm_gen_mxfp8_block_scale_moe(
|
||||
beta=0.0,
|
||||
limit=None,
|
||||
act_type="mxfp8",
|
||||
is_gated=is_gated,
|
||||
activation="swiglu" if is_gated else "relu2",
|
||||
use_interleaved_layout=False,
|
||||
)
|
||||
|
||||
# Shuffle weights/scales with the same indexed layout used by TRTLLM kernels.
|
||||
@@ -1150,3 +1205,328 @@ def test_trtllm_gen_mxfp8_block_scale_moe(
|
||||
|
||||
# Block-scale MXFP8 kernels are approximate; require majority close.
|
||||
check_accuracy(ref, out, atol=0.1, rtol=0.85, percent=0.8)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# ROCm Oracle-based kernel execution tests
|
||||
# -----------------------------------------------------------------------------
|
||||
# TODO: Further tighten the accuracy threshold.
|
||||
# - More accurate ref moe to include activation quantization
|
||||
# - Check aiter kernel accuracy. E.g., quant / dequant details.
|
||||
ROCM_BACKEND_CONFIGS = {
|
||||
"TRITON": {
|
||||
"activation": "SWIGLUOAI",
|
||||
"rtol": 0.3,
|
||||
"percent": 0.95,
|
||||
"requires_aiter": False,
|
||||
"requires_gfx950": False,
|
||||
},
|
||||
"TRITON_UNFUSED": {
|
||||
"activation": "SWIGLUOAI",
|
||||
"rtol": 0.3,
|
||||
"percent": 0.95,
|
||||
"requires_aiter": False,
|
||||
"requires_gfx950": False,
|
||||
},
|
||||
"AITER_MXFP4_BF16": {
|
||||
"activation": "SILU",
|
||||
"rtol": 1.0,
|
||||
"percent": 0.7,
|
||||
"requires_aiter": True,
|
||||
"requires_gfx950": True,
|
||||
},
|
||||
"AITER_MXFP4_FP8": {
|
||||
"activation": "SWIGLUOAI",
|
||||
"rtol": 0.5,
|
||||
"percent": 0.9,
|
||||
"requires_aiter": True,
|
||||
"requires_gfx950": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend_name", list(ROCM_BACKEND_CONFIGS.keys()))
|
||||
@pytest.mark.parametrize("topk", [4])
|
||||
@pytest.mark.parametrize("num_experts", [8])
|
||||
@pytest.mark.parametrize("num_tokens,hidden_size,intermediate_size", [(16, 256, 256)])
|
||||
@pytest.mark.skipif(
|
||||
not ROCM_AVAILABLE,
|
||||
reason="ROCm is required for this test",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp4_moe_oracle(
|
||||
backend_name: str,
|
||||
topk: int,
|
||||
num_experts: int,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
):
|
||||
"""
|
||||
Test ROCm MXFP4 MoE using oracle functions.
|
||||
|
||||
This test validates that the oracle functions work end-to-end:
|
||||
- select_mxfp4_moe_backend() selects a valid backend
|
||||
- convert_to_mxfp4_moe_kernel_format() converts weights without error
|
||||
- make_mxfp4_moe_quant_config() builds a valid quant config
|
||||
- make_mxfp4_moe_kernel() creates a kernel that runs without error
|
||||
- The kernel output is within accuracy tolerance of reference
|
||||
"""
|
||||
config = ROCM_BACKEND_CONFIGS[backend_name]
|
||||
|
||||
# Check platform requirements
|
||||
if not ROCM_TRITON_KERNELS_AVAILABLE:
|
||||
pytest.skip("triton_kernels required for quantization")
|
||||
if config["requires_aiter"] and not ROCM_AITER_AVAILABLE:
|
||||
pytest.skip(f"Backend {backend_name} requires AITER")
|
||||
if config["requires_gfx950"] and not ROCM_GFX950:
|
||||
pytest.skip(f"Backend {backend_name} requires GFX950")
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
backend_to_kernel_cls,
|
||||
convert_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
make_mxfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
# Initialize workspace manager (needed for modular kernels)
|
||||
init_workspace_manager(torch.accelerator.current_device_index())
|
||||
|
||||
# Map string to enum
|
||||
backend = Mxfp4MoeBackend[backend_name]
|
||||
|
||||
# Get experts class from oracle
|
||||
experts_cls_list = backend_to_kernel_cls(backend)
|
||||
if experts_cls_list is None or len(experts_cls_list) == 0:
|
||||
pytest.skip(f"Backend {backend_name} not available")
|
||||
|
||||
# Use first experts class
|
||||
experts_cls = experts_cls_list[0]
|
||||
|
||||
torch.manual_seed(42)
|
||||
dtype = torch.bfloat16
|
||||
device = "cuda:0"
|
||||
|
||||
# Create MoE config with Renormalize routing (required by monolithic kernels)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=hidden_size,
|
||||
intermediate_size_per_partition=intermediate_size,
|
||||
num_local_experts=num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation[config["activation"]],
|
||||
in_dtype=dtype,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
)
|
||||
|
||||
# Create float weights in checkpoint format:
|
||||
# w13: [num_experts, 2*intermediate_size, hidden_size]
|
||||
# w2: [num_experts, hidden_size, intermediate_size]
|
||||
w13_float = torch.randn(
|
||||
num_experts, 2 * intermediate_size, hidden_size, dtype=dtype, device=device
|
||||
)
|
||||
w2_float = torch.randn(
|
||||
num_experts, hidden_size, intermediate_size, dtype=dtype, device=device
|
||||
)
|
||||
|
||||
# dynamic_mxfp4_quant expects 2D input, so reshape 3D weights
|
||||
# w13: [E, 2*I, H] -> [E*2*I, H] -> quantize -> [E, 2*I, H//2]
|
||||
# w2: [E, H, I] -> [E*H, I] -> quantize -> [E, H, I//2]
|
||||
w13_2d = w13_float.reshape(-1, hidden_size)
|
||||
w13_quant_2d, w13_scale_2d = dynamic_mxfp4_quant(w13_2d)
|
||||
w13_quant = w13_quant_2d.reshape(num_experts, 2 * intermediate_size, -1)
|
||||
w13_scale = w13_scale_2d.reshape(num_experts, 2 * intermediate_size, -1)
|
||||
|
||||
w2_2d = w2_float.reshape(-1, intermediate_size)
|
||||
w2_quant_2d, w2_scale_2d = dynamic_mxfp4_quant(w2_2d)
|
||||
w2_quant = w2_quant_2d.reshape(num_experts, hidden_size, -1)
|
||||
w2_scale = w2_scale_2d.reshape(num_experts, hidden_size, -1)
|
||||
|
||||
w13_bias = torch.randn(
|
||||
num_experts, 2 * intermediate_size, dtype=dtype, device=device
|
||||
)
|
||||
w2_bias = torch.randn(num_experts, hidden_size, dtype=dtype, device=device)
|
||||
|
||||
# Create static input scales for W4A8 backend (AITER_MXFP4_FP8)
|
||||
w13_input_scale: torch.Tensor | None = None
|
||||
w2_input_scale: torch.Tensor | None = None
|
||||
if backend_name == "AITER_MXFP4_FP8":
|
||||
# Static FP8 scales: one scale per expert
|
||||
w13_input_scale = torch.ones(num_experts, dtype=torch.float32, device=device)
|
||||
w2_input_scale = torch.ones(num_experts, dtype=torch.float32, device=device)
|
||||
|
||||
# Create mock layer for oracle functions
|
||||
class MockLayer:
|
||||
w13_weight: torch.Tensor
|
||||
w2_weight: torch.Tensor
|
||||
w13_weight_scale: torch.Tensor
|
||||
w2_weight_scale: torch.Tensor
|
||||
w13_input_scale: torch.Tensor | None
|
||||
w2_input_scale: torch.Tensor | None
|
||||
|
||||
layer = MockLayer()
|
||||
layer.w13_weight = w13_quant
|
||||
layer.w2_weight = w2_quant
|
||||
layer.w13_weight_scale = w13_scale
|
||||
layer.w2_weight_scale = w2_scale
|
||||
layer.w13_input_scale = w13_input_scale
|
||||
layer.w2_input_scale = w2_input_scale
|
||||
|
||||
# Convert weights using oracle
|
||||
w13_conv, w2_conv, w13_scale_conv, w2_scale_conv, w13_bias_conv, w2_bias_conv = (
|
||||
convert_to_mxfp4_moe_kernel_format(
|
||||
mxfp4_backend=backend,
|
||||
layer=layer, # type: ignore[arg-type]
|
||||
w13_weight=w13_quant,
|
||||
w2_weight=w2_quant,
|
||||
w13_weight_scale=w13_scale,
|
||||
w2_weight_scale=w2_scale,
|
||||
w13_bias=w13_bias,
|
||||
w2_bias=w2_bias,
|
||||
)
|
||||
)
|
||||
|
||||
# Build quant config using oracle
|
||||
quant_config = make_mxfp4_moe_quant_config(
|
||||
mxfp4_backend=backend,
|
||||
w1_scale=w13_scale_conv,
|
||||
w2_scale=w2_scale_conv,
|
||||
w1_bias=w13_bias_conv,
|
||||
w2_bias=w2_bias_conv,
|
||||
a1_scale=w13_input_scale,
|
||||
a2_scale=w2_input_scale,
|
||||
)
|
||||
|
||||
# Select activation based on backend
|
||||
activation_name = str(config["activation"])
|
||||
activation = MoEActivation[activation_name]
|
||||
|
||||
# Build kernel using oracle
|
||||
assert quant_config is not None, "Failed to create quant config"
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=quant_config,
|
||||
moe_config=moe_config,
|
||||
mxfp4_backend=backend,
|
||||
experts_cls=experts_cls,
|
||||
routing_tables=None,
|
||||
shared_experts=None,
|
||||
)
|
||||
|
||||
# Create inputs
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
router_logits = torch.randn(
|
||||
num_tokens, num_experts, dtype=torch.float32, device=device
|
||||
)
|
||||
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1, sorted=True)
|
||||
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
|
||||
|
||||
# Run kernel - use appropriate method based on impl type
|
||||
if kernel.is_monolithic:
|
||||
# Monolithic impl uses router_logits
|
||||
out = kernel.apply_monolithic(
|
||||
hidden_states=x,
|
||||
w1=w13_conv,
|
||||
w2=w2_conv,
|
||||
router_logits=router_logits,
|
||||
activation=activation,
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
else:
|
||||
# Modular impl uses topk_weights and topk_ids
|
||||
out = kernel.apply(
|
||||
hidden_states=x,
|
||||
w1=w13_conv,
|
||||
w2=w2_conv,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
# Verify output is valid (no NaN/Inf) and has expected shape
|
||||
assert out.shape == (num_tokens, hidden_size), f"Unexpected shape: {out.shape}"
|
||||
assert not torch.any(torch.isnan(out)), "Output contains NaN"
|
||||
assert not torch.any(torch.isinf(out)), "Output contains Inf"
|
||||
|
||||
# Verify output has reasonable magnitude (not all zeros)
|
||||
assert out.abs().max() > 0.01, "Output is effectively zero"
|
||||
|
||||
# Dequantize weights for reference computation
|
||||
w13_dq = upcast_from_mxfp(
|
||||
w13_quant.view(torch.uint8), w13_scale, torch.bfloat16, axis=-1
|
||||
)
|
||||
w2_dq = upcast_from_mxfp(
|
||||
w2_quant.view(torch.uint8), w2_scale, torch.bfloat16, axis=-1
|
||||
)
|
||||
|
||||
# Determine activation type and layout
|
||||
# SWIGLUOAI uses interleaved layout (gate/up alternating)
|
||||
# SILU uses chunked layout (first half gate, second half up)
|
||||
use_interleaved = activation == MoEActivation.SWIGLUOAI
|
||||
if activation in [MoEActivation.SWIGLUOAI, MoEActivation.SILU]:
|
||||
act_name = "swiglu"
|
||||
else:
|
||||
act_name = "relu2"
|
||||
|
||||
ref = reference_moe(
|
||||
router_logits,
|
||||
topk,
|
||||
num_experts,
|
||||
x.to(torch.float32),
|
||||
w13_dq.to(torch.float32),
|
||||
w13_bias.to(torch.float32),
|
||||
w2_dq.to(torch.float32),
|
||||
w2_bias.to(torch.float32),
|
||||
alpha=1.702 if activation == MoEActivation.SWIGLUOAI else 1.0,
|
||||
beta=1.0 if activation == MoEActivation.SWIGLUOAI else 0.0,
|
||||
limit=7.0 if activation == MoEActivation.SWIGLUOAI else None,
|
||||
act_type="bf16",
|
||||
activation=act_name,
|
||||
use_interleaved_layout=use_interleaved,
|
||||
)
|
||||
|
||||
# Compute and print accuracy statistics
|
||||
diff = (ref.float() - out.float()).abs()
|
||||
rel_diff = diff / (ref.float().abs() + 1e-6)
|
||||
|
||||
print(f"\n[{backend_name}] Accuracy statistics:")
|
||||
print(
|
||||
f" Reference: min={ref.min():.4f}, max={ref.max():.4f}, mean={ref.mean():.4f}"
|
||||
)
|
||||
print(
|
||||
f" Output: min={out.min():.4f}, max={out.max():.4f}, mean={out.mean():.4f}"
|
||||
)
|
||||
print(
|
||||
f" Abs diff: min={diff.min():.4f}, max={diff.max():.4f}, "
|
||||
f"mean={diff.mean():.4f}"
|
||||
)
|
||||
print(
|
||||
f" Rel diff: min={rel_diff.min():.4f}, max={rel_diff.max():.4f}, "
|
||||
f"mean={rel_diff.mean():.4f}"
|
||||
)
|
||||
|
||||
# Check what percentage of values are within various tolerances
|
||||
for rtol in [0.1, 0.5, 1.0, 2.0]:
|
||||
within_tol = (diff <= rtol * out.float().abs()).float().mean()
|
||||
print(f" Within rtol={rtol}: {within_tol * 100:.1f}%")
|
||||
|
||||
# Check accuracy using per-backend thresholds
|
||||
check_accuracy(ref, out, atol=0.1, rtol=config["rtol"], percent=config["percent"])
|
||||
|
||||
Reference in New Issue
Block a user