[Bugfix][Kernel] Skip the fused silu-mul block-quant fast path when a swiglu clamp is set (#57984)
Signed-off-by: Garrett Goon <garrett@primeintellect.ai> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
8644d2af2f
commit
8bf051cd51
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
|||||||
from vllm.model_executor.layers.fused_moe.config import (
|
from vllm.model_executor.layers.fused_moe.config import (
|
||||||
fp8_w8a8_moe_quant_config,
|
fp8_w8a8_moe_quant_config,
|
||||||
)
|
)
|
||||||
|
from vllm.model_executor.layers.fused_moe.experts import triton_moe
|
||||||
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
|
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
|
||||||
_valid_deep_gemm_shape,
|
_valid_deep_gemm_shape,
|
||||||
)
|
)
|
||||||
@@ -246,6 +247,60 @@ def test_w8a8_block_fp8_fused_moe(
|
|||||||
torch.testing.assert_close(m_out, ref_out_m, atol=tol, rtol=tol)
|
torch.testing.assert_close(m_out, ref_out_m, atol=tol, rtol=tol)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
|
||||||
|
@torch.inference_mode()
|
||||||
|
def test_w8a8_block_fp8_clamp_skips_fused_silu_quant(
|
||||||
|
clamp_limit, monkeypatch, workspace_init, disable_deepgemm_ue8m0
|
||||||
|
):
|
||||||
|
"""A SwiGLU clamp must bypass the fused SiLU+quant kernel, which cannot clamp."""
|
||||||
|
torch.manual_seed(0)
|
||||||
|
M, N, K, E, topk = 16, 256, 512, 8, 2
|
||||||
|
block_size = [128, 128]
|
||||||
|
|
||||||
|
a = torch.randn((M, K), dtype=torch.bfloat16) / 10
|
||||||
|
score = torch.randn((M, E), dtype=torch.bfloat16)
|
||||||
|
(_, w1, w1_s, _), (_, w2, w2_s, _) = make_test_weights(
|
||||||
|
E, N, K, torch.bfloat16, torch.float8_e4m3fn, block_shape=block_size
|
||||||
|
)
|
||||||
|
quant_config = fp8_w8a8_moe_quant_config(
|
||||||
|
w1_scale=w1_s,
|
||||||
|
w2_scale=w2_s,
|
||||||
|
block_shape=block_size,
|
||||||
|
gemm1_clamp_limit=clamp_limit,
|
||||||
|
)
|
||||||
|
m_fused_moe = modular_triton_fused_moe(make_dummy_moe_config(), quant_config)
|
||||||
|
topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False)
|
||||||
|
|
||||||
|
fused_calls = 0
|
||||||
|
fused_op = triton_moe.ops.silu_and_mul_per_block_quant
|
||||||
|
|
||||||
|
def counting_fused_op(*args, **kwargs):
|
||||||
|
nonlocal fused_calls
|
||||||
|
fused_calls += 1
|
||||||
|
return fused_op(*args, **kwargs)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
triton_moe.ops, "silu_and_mul_per_block_quant", counting_fused_op
|
||||||
|
)
|
||||||
|
|
||||||
|
with set_current_vllm_config(vllm_config):
|
||||||
|
m_fused_moe.apply(
|
||||||
|
a,
|
||||||
|
w1,
|
||||||
|
w2,
|
||||||
|
topk_weights,
|
||||||
|
topk_ids,
|
||||||
|
activation=MoEActivation.SILU,
|
||||||
|
apply_router_weight_on_input=False,
|
||||||
|
expert_map=None,
|
||||||
|
global_num_experts=E,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (fused_calls > 0) == (clamp_limit is None), (
|
||||||
|
f"fused fast path ran {fused_calls} times with clamp_limit={clamp_limit}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS_DG)
|
@pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS_DG)
|
||||||
@pytest.mark.parametrize("E", E)
|
@pytest.mark.parametrize("E", E)
|
||||||
@pytest.mark.parametrize("topk", TOP_KS)
|
@pytest.mark.parametrize("topk", TOP_KS)
|
||||||
|
|||||||
@@ -479,8 +479,11 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
|||||||
# Fuse SiLU+Mul + FP8 block quantize into a single kernel
|
# Fuse SiLU+Mul + FP8 block quantize into a single kernel
|
||||||
# when conditions permit (gated SiLU, fp8 block quant with
|
# when conditions permit (gated SiLU, fp8 block quant with
|
||||||
# group_size=128, no LoRA requiring the BF16 intermediate).
|
# group_size=128, no LoRA requiring the BF16 intermediate).
|
||||||
|
# The fused kernel has no clamp parameter, so a configured
|
||||||
|
# SwiGLU clamp limit falls through to the unfused path.
|
||||||
if (
|
if (
|
||||||
activation == MoEActivation.SILU
|
activation == MoEActivation.SILU
|
||||||
|
and self.activation_config.clamp_limit is None
|
||||||
and self.quant_config.use_fp8_w8a8
|
and self.quant_config.use_fp8_w8a8
|
||||||
and self.block_shape == [128, 128]
|
and self.block_shape == [128, 128]
|
||||||
and lora_context is None
|
and lora_context is None
|
||||||
|
|||||||
Reference in New Issue
Block a user