Refactor AWQ Marlin MoE onto modular WNA16 oracle (#42483)
Signed-off-by: Siddharth Bedekar <bedeksid@gmail.com> Signed-off-by: Siddharth Bedekar <104613085+bedeks@users.noreply.github.com> Co-authored-by: Robert Shaw <robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Robert Shaw
OpenAI Codex
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f5d3dc7115
commit
78e7a7b9b0
@@ -1338,36 +1338,96 @@ def test_cpu_fused_moe_basic(
|
||||
torch.testing.assert_close(out, ref, atol=atol, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [16, 32, 64])
|
||||
@pytest.mark.parametrize("n", [128])
|
||||
@pytest.mark.parametrize("k", [128])
|
||||
@pytest.mark.parametrize("e", [8, 12, 16, 32])
|
||||
@pytest.mark.parametrize("topk", [2, 4])
|
||||
@pytest.mark.parametrize("max_tokens_per_batch", [16, 32, 64])
|
||||
def _batched_fused_marlin_moe_cases() -> list[Any]:
|
||||
cases = [
|
||||
pytest.param(
|
||||
m,
|
||||
128,
|
||||
128,
|
||||
e,
|
||||
topk,
|
||||
max_tokens_per_batch,
|
||||
torch.bfloat16,
|
||||
scalar_types.float4_e2m1f,
|
||||
None,
|
||||
1e-3,
|
||||
id=(
|
||||
f"m{m}-n128-k128-e{e}-topk{topk}-max_tokens{max_tokens_per_batch}-mxfp4"
|
||||
),
|
||||
)
|
||||
for m in [16, 32, 64]
|
||||
for e in [8, 12, 16, 32]
|
||||
for topk in [2, 4]
|
||||
for max_tokens_per_batch in [16, 32, 64]
|
||||
]
|
||||
cases.append(
|
||||
pytest.param(
|
||||
32,
|
||||
128,
|
||||
128,
|
||||
8,
|
||||
2,
|
||||
64,
|
||||
torch.float16,
|
||||
scalar_types.uint4,
|
||||
scalar_types.int8,
|
||||
4e-2,
|
||||
id="awq-int8-activation-metadata",
|
||||
)
|
||||
)
|
||||
return cases
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("m,n,k,e,topk,max_tokens_per_batch,dtype,quant_dtype,input_type,atol"),
|
||||
_batched_fused_marlin_moe_cases(),
|
||||
)
|
||||
@pytest.mark.skipif(current_platform.is_rocm(), reason="Skip for rocm")
|
||||
def test_batched_fused_marlin_moe(
|
||||
m: int, n: int, k: int, e: int, topk: int, max_tokens_per_batch: int
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
e: int,
|
||||
topk: int,
|
||||
max_tokens_per_batch: int,
|
||||
dtype: torch.dtype,
|
||||
quant_dtype: ScalarType,
|
||||
input_type: ScalarType | None,
|
||||
atol: float,
|
||||
):
|
||||
print(
|
||||
f"testing m={m}, n={n}, k={k}, e={e}, "
|
||||
f"topk={topk}, "
|
||||
f"max_tokens_per_batch={max_tokens_per_batch}"
|
||||
f"max_tokens_per_batch={max_tokens_per_batch}, "
|
||||
f"dtype={dtype}, quant_dtype={quant_dtype}, input_type={input_type}"
|
||||
)
|
||||
set_random_seed(0)
|
||||
|
||||
dtype = torch.bfloat16
|
||||
quant_dtype = scalar_types.float4_e2m1f
|
||||
group_size = 32
|
||||
if input_type == scalar_types.int8:
|
||||
input_dtype = torch.int8
|
||||
elif input_type == scalar_types.float8_e4m3fn:
|
||||
input_dtype = torch.float8_e4m3fn
|
||||
else:
|
||||
input_dtype = None
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 20
|
||||
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 20
|
||||
|
||||
w1_data = MarlinMoEWeightData.make(
|
||||
w=w1, quant_type=quant_dtype, group_size=group_size, act_order=None
|
||||
w=w1,
|
||||
quant_type=quant_dtype,
|
||||
group_size=group_size,
|
||||
act_order=None,
|
||||
input_type=input_type,
|
||||
)
|
||||
w2_data = MarlinMoEWeightData.make(
|
||||
w=w2, quant_type=quant_dtype, group_size=group_size, act_order=None
|
||||
w=w2,
|
||||
quant_type=quant_dtype,
|
||||
group_size=group_size,
|
||||
act_order=None,
|
||||
input_type=input_type,
|
||||
)
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
@@ -1487,6 +1547,12 @@ def test_batched_fused_marlin_moe(
|
||||
"quant_type_id": quant_dtype.id,
|
||||
"is_k_full": True,
|
||||
}
|
||||
if input_dtype is not None:
|
||||
kwargs["input_dtype"] = input_dtype
|
||||
if w1_data.a_scales_factor is not None:
|
||||
kwargs["input_global_scale1"] = w1_data.a_scales_factor
|
||||
if w2_data.a_scales_factor is not None:
|
||||
kwargs["input_global_scale2"] = w2_data.a_scales_factor
|
||||
|
||||
# Reference
|
||||
fused_marlin_moe_kwargs = kwargs | {
|
||||
@@ -1502,7 +1568,7 @@ def test_batched_fused_marlin_moe(
|
||||
pytest.skip("Cannot represent data in Batched Format.")
|
||||
marlin_output = br.run(a, kwargs)
|
||||
|
||||
torch.testing.assert_close(marlin_output, ref_marlin_output, atol=1e-3, rtol=0)
|
||||
torch.testing.assert_close(marlin_output, ref_marlin_output, atol=atol, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", [(32, 1024, 1024)])
|
||||
|
||||
Reference in New Issue
Block a user