[Quantization] Add per-token NVFP4 CuTe-DSL MoE backend (#50030)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: S1ro1 <matej.sirovatka@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
This commit is contained in:
co-authored by
aoshen02
OpenAI Codex
aoshen02
parent
d5ee309314
commit
0103a9b96f
@@ -605,6 +605,30 @@ def test_checkpoint_quantization_rejects_online_shorthand(tmp_path) -> None:
|
||||
ModelConfig(model=str(tmp_path), quantization="fp8_per_channel")
|
||||
|
||||
|
||||
def test_nvfp4_per_token_backend_contract() -> None:
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutedsl_moe import (
|
||||
FlashInferCuteDSLExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
|
||||
TrtLlmNvFp4ExpertsModular,
|
||||
TrtLlmNvFp4ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kNvfp4DynamicToken,
|
||||
kNvfp4Static,
|
||||
)
|
||||
|
||||
scheme = (kNvfp4Static, kNvfp4DynamicToken)
|
||||
assert TrtLlmNvFp4ExpertsMonolithic._supports_quant_scheme(*scheme)
|
||||
assert TrtLlmNvFp4ExpertsModular._supports_quant_scheme(*scheme)
|
||||
assert FlashInferCuteDSLExperts._supports_quant_scheme(*scheme)
|
||||
assert FlashInferCuteDSLExperts._supports_no_act_and_mul()
|
||||
assert not FlashInferExperts._supports_quant_scheme(*scheme)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("per_token_activation", [False, True])
|
||||
@pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16])
|
||||
def test_nvfp4_one_sided_sizes_dispatched_activations(
|
||||
|
||||
@@ -11,8 +11,8 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
align_fp4_moe_hidden_dim_for_fi,
|
||||
align_fp4_moe_weights_for_fi,
|
||||
align_trtllm_fp4_moe_hidden_dim_for_fi,
|
||||
)
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ def test_align_trtllm_fp4_moe_hidden_dim_noop():
|
||||
w2_scale = torch.arange(2 * 512 * 1, dtype=torch.uint8).reshape(2, 512, 1)
|
||||
|
||||
out_w13, out_w13_scale, out_w2, out_w2_scale, padded_hidden = (
|
||||
align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
|
||||
align_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
|
||||
)
|
||||
|
||||
assert padded_hidden == 512
|
||||
@@ -90,7 +90,7 @@ def test_align_trtllm_fp4_moe_hidden_dim_pads_to_256_multiple():
|
||||
)
|
||||
|
||||
out_w13, out_w13_scale, out_w2, out_w2_scale, out_hidden_dim = (
|
||||
align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
|
||||
align_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
|
||||
)
|
||||
|
||||
assert out_hidden_dim == padded_hidden_dim
|
||||
|
||||
Reference in New Issue
Block a user