Refactor AWQ Marlin MoE onto modular WNA16 oracle (#42483)

Signed-off-by: Siddharth Bedekar <bedeksid@gmail.com>
Signed-off-by: Siddharth Bedekar <104613085+bedeks@users.noreply.github.com>
Co-authored-by: Robert Shaw <robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Siddharth Bedekar
2026-05-18 08:02:43 -07:00
committed by GitHub
co-authored by Robert Shaw OpenAI Codex gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f5d3dc7115
commit 78e7a7b9b0
6 changed files with 405 additions and 206 deletions
+79 -13
View File
@@ -1338,36 +1338,96 @@ def test_cpu_fused_moe_basic(
torch.testing.assert_close(out, ref, atol=atol, rtol=0)
@pytest.mark.parametrize("m", [16, 32, 64])
@pytest.mark.parametrize("n", [128])
@pytest.mark.parametrize("k", [128])
@pytest.mark.parametrize("e", [8, 12, 16, 32])
@pytest.mark.parametrize("topk", [2, 4])
@pytest.mark.parametrize("max_tokens_per_batch", [16, 32, 64])
def _batched_fused_marlin_moe_cases() -> list[Any]:
cases = [
pytest.param(
m,
128,
128,
e,
topk,
max_tokens_per_batch,
torch.bfloat16,
scalar_types.float4_e2m1f,
None,
1e-3,
id=(
f"m{m}-n128-k128-e{e}-topk{topk}-max_tokens{max_tokens_per_batch}-mxfp4"
),
)
for m in [16, 32, 64]
for e in [8, 12, 16, 32]
for topk in [2, 4]
for max_tokens_per_batch in [16, 32, 64]
]
cases.append(
pytest.param(
32,
128,
128,
8,
2,
64,
torch.float16,
scalar_types.uint4,
scalar_types.int8,
4e-2,
id="awq-int8-activation-metadata",
)
)
return cases
@pytest.mark.parametrize(
("m,n,k,e,topk,max_tokens_per_batch,dtype,quant_dtype,input_type,atol"),
_batched_fused_marlin_moe_cases(),
)
@pytest.mark.skipif(current_platform.is_rocm(), reason="Skip for rocm")
def test_batched_fused_marlin_moe(
m: int, n: int, k: int, e: int, topk: int, max_tokens_per_batch: int
m: int,
n: int,
k: int,
e: int,
topk: int,
max_tokens_per_batch: int,
dtype: torch.dtype,
quant_dtype: ScalarType,
input_type: ScalarType | None,
atol: float,
):
print(
f"testing m={m}, n={n}, k={k}, e={e}, "
f"topk={topk}, "
f"max_tokens_per_batch={max_tokens_per_batch}"
f"max_tokens_per_batch={max_tokens_per_batch}, "
f"dtype={dtype}, quant_dtype={quant_dtype}, input_type={input_type}"
)
set_random_seed(0)
dtype = torch.bfloat16
quant_dtype = scalar_types.float4_e2m1f
group_size = 32
if input_type == scalar_types.int8:
input_dtype = torch.int8
elif input_type == scalar_types.float8_e4m3fn:
input_dtype = torch.float8_e4m3fn
else:
input_dtype = None
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 20
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 20
w1_data = MarlinMoEWeightData.make(
w=w1, quant_type=quant_dtype, group_size=group_size, act_order=None
w=w1,
quant_type=quant_dtype,
group_size=group_size,
act_order=None,
input_type=input_type,
)
w2_data = MarlinMoEWeightData.make(
w=w2, quant_type=quant_dtype, group_size=group_size, act_order=None
w=w2,
quant_type=quant_dtype,
group_size=group_size,
act_order=None,
input_type=input_type,
)
score = torch.randn((m, e), device="cuda", dtype=dtype)
@@ -1487,6 +1547,12 @@ def test_batched_fused_marlin_moe(
"quant_type_id": quant_dtype.id,
"is_k_full": True,
}
if input_dtype is not None:
kwargs["input_dtype"] = input_dtype
if w1_data.a_scales_factor is not None:
kwargs["input_global_scale1"] = w1_data.a_scales_factor
if w2_data.a_scales_factor is not None:
kwargs["input_global_scale2"] = w2_data.a_scales_factor
# Reference
fused_marlin_moe_kwargs = kwargs | {
@@ -1502,7 +1568,7 @@ def test_batched_fused_marlin_moe(
pytest.skip("Cannot represent data in Batched Format.")
marlin_output = br.run(a, kwargs)
torch.testing.assert_close(marlin_output, ref_marlin_output, atol=1e-3, rtol=0)
torch.testing.assert_close(marlin_output, ref_marlin_output, atol=atol, rtol=0)
@pytest.mark.parametrize("m,n,k", [(32, 1024, 1024)])