[Kernel] Add b12x FP4 MoE backend (#52018)
Signed-off-by: Luke Alonso <lalonso@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -43,6 +43,7 @@ def dequantize_nvfp4_to_dtype(
|
||||
device,
|
||||
block_size=16,
|
||||
is_sf_128x4_layout=True,
|
||||
is_sf_linear_layout=False,
|
||||
):
|
||||
"""Dequantize the fp4 tensor back to high precision."""
|
||||
# Two fp4 values are packed into one uint8.
|
||||
@@ -52,7 +53,9 @@ def dequantize_nvfp4_to_dtype(
|
||||
tensor_f32 = break_fp4_bytes(tensor_fp4, dtype)
|
||||
tensor_f32 = tensor_f32.reshape(m, k // block_size, block_size)
|
||||
tensor_sf = tensor_sf.view(torch.float8_e4m3fn)
|
||||
if is_sf_128x4_layout:
|
||||
if is_sf_linear_layout:
|
||||
tensor_sf = tensor_sf[:m, : k // block_size]
|
||||
elif is_sf_128x4_layout:
|
||||
tensor_sf = convert_swizzled_to_linear(tensor_sf, m, k, block_size)
|
||||
else:
|
||||
tensor_sf = convert_swizzled_8x4_layout_to_linear(tensor_sf, m, k, block_size)
|
||||
@@ -146,7 +149,15 @@ def get_nvfp4_global_scale(a: torch.Tensor):
|
||||
return (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.abs(a).max().to(torch.float32)
|
||||
|
||||
|
||||
def quant_nvfp4_tensor(a: torch.Tensor):
|
||||
def quant_nvfp4_tensor(
|
||||
a: torch.Tensor,
|
||||
*,
|
||||
is_sf_swizzled_layout: bool = True,
|
||||
):
|
||||
a_global_scale = get_nvfp4_global_scale(a)
|
||||
a_quant, a_block_scale = scaled_fp4_quant(a, a_global_scale)
|
||||
a_quant, a_block_scale = scaled_fp4_quant(
|
||||
a,
|
||||
a_global_scale,
|
||||
is_sf_swizzled_layout=is_sf_swizzled_layout,
|
||||
)
|
||||
return a_quant, a_block_scale, a_global_scale
|
||||
|
||||
@@ -181,8 +181,14 @@ def test_b12x_warmup_deduplicates_registered_signatures(monkeypatch) -> None:
|
||||
|
||||
worker = SimpleNamespace(
|
||||
get_model=lambda: SimpleNamespace(modules=modules),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
||||
scheduler_config=SimpleNamespace(
|
||||
max_num_batched_tokens=8,
|
||||
max_num_scheduled_tokens=32,
|
||||
),
|
||||
model_config=SimpleNamespace(dtype=torch.float32),
|
||||
vllm_config=SimpleNamespace(
|
||||
compilation_config=SimpleNamespace(compile_sizes=[4, 16])
|
||||
),
|
||||
)
|
||||
platform = SimpleNamespace(
|
||||
is_cuda=lambda: True,
|
||||
@@ -200,7 +206,7 @@ def test_b12x_warmup_deduplicates_registered_signatures(monkeypatch) -> None:
|
||||
|
||||
assert scans == 1
|
||||
assert calls == [
|
||||
("first", (1, 2, 8), torch.bfloat16),
|
||||
("second", (1, 2, 8), torch.bfloat16),
|
||||
("first", (1, 2, 4, 8, 16, 32), torch.bfloat16),
|
||||
("second", (1, 2, 4, 8, 16, 32), torch.bfloat16),
|
||||
]
|
||||
assert synchronized == [True]
|
||||
|
||||
@@ -663,12 +663,28 @@ def test_inc_mxfp4_linear_method_registers_and_processes_weights(
|
||||
assert captured["processed_layer"] is layer
|
||||
|
||||
|
||||
def test_inc_mxfp4_moe_method_registers_weights_and_builds_kernel(
|
||||
@pytest.mark.parametrize(
|
||||
("moe_backend", "is_xpu", "mxfp4_backend"),
|
||||
[
|
||||
("auto", True, Mxfp4MoeBackend.XPU),
|
||||
("b12x", False, Mxfp4MoeBackend.B12X_MXFP4_MXFP8),
|
||||
],
|
||||
)
|
||||
def test_inc_mxfp4_moe_method_preserves_checkpoint_packing(
|
||||
monkeypatch,
|
||||
moe_backend: str,
|
||||
is_xpu: bool,
|
||||
mxfp4_backend: Mxfp4MoeBackend,
|
||||
) -> None:
|
||||
captured = {}
|
||||
expected_quant_config = object()
|
||||
expected_kernel = object()
|
||||
expected_kernel = SimpleNamespace(
|
||||
fused_experts=SimpleNamespace(
|
||||
process_weights_after_loading=lambda layer: captured.update(
|
||||
{"processed_layer": layer}
|
||||
)
|
||||
)
|
||||
)
|
||||
expected_experts_cls = object()
|
||||
|
||||
monkeypatch.setattr(
|
||||
@@ -676,11 +692,16 @@ def test_inc_mxfp4_moe_method_registers_weights_and_builds_kernel(
|
||||
"CutlassExpertsMxfp4._supports_current_device",
|
||||
lambda: False,
|
||||
)
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: is_xpu)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe."
|
||||
"select_mxfp4_moe_backend",
|
||||
lambda moe: (Mxfp4MoeBackend.XPU, expected_experts_cls),
|
||||
lambda moe: (mxfp4_backend, expected_experts_cls),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe."
|
||||
"prepare_moe_fp4_layer_for_marlin",
|
||||
lambda layer: pytest.fail("packed backends must not use Marlin packing"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe."
|
||||
@@ -698,7 +719,10 @@ def test_inc_mxfp4_moe_method_registers_weights_and_builds_kernel(
|
||||
INCMxfp4MoEMethod,
|
||||
)
|
||||
|
||||
expected_moe_config = SimpleNamespace(w13_num_shards=2)
|
||||
expected_moe_config = SimpleNamespace(
|
||||
w13_num_shards=2,
|
||||
moe_backend=moe_backend,
|
||||
)
|
||||
method = INCMxfp4MoEMethod(moe=cast(Any, expected_moe_config))
|
||||
layer = torch.nn.Module()
|
||||
layer._expert_routing_tables = lambda: "routing-tables"
|
||||
@@ -731,6 +755,7 @@ def test_inc_mxfp4_moe_method_registers_weights_and_builds_kernel(
|
||||
assert captured["kernel_kwargs"]["moe_config"] is expected_moe_config
|
||||
assert captured["kernel_kwargs"]["experts_cls"] is expected_experts_cls
|
||||
assert captured["kernel_kwargs"]["routing_tables"] == "routing-tables"
|
||||
assert captured["processed_layer"] is layer
|
||||
assert method.moe_kernel is expected_kernel
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user