[MoE Refactor][15/N] Apply Refactor to Fp8 (#31415)
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"
|
||||
accuracy_threshold: 0.92
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
|
||||
accuracy_threshold: 0.88
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_high_throughput"
|
||||
env:
|
||||
VLLM_USE_DEEP_GEMM: "1"
|
||||
VLLM_USE_DEEP_GEMM_MOE: "1"
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
|
||||
accuracy_threshold: 0.88
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --disable-uvicorn-access-log"
|
||||
env:
|
||||
VLLM_USE_DEEP_GEMM: "1"
|
||||
VLLM_USE_DEEP_GEMM_MOE: "1"
|
||||
VLLM_USE_DEEP_GEMM_E8M0: "0"
|
||||
@@ -0,0 +1,8 @@
|
||||
model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
|
||||
accuracy_threshold: 0.88
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"
|
||||
env:
|
||||
VLLM_USE_DEEP_GEMM: "1"
|
||||
VLLM_USE_DEEP_GEMM_MOE: "1"
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
|
||||
accuracy_threshold: 0.85
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_high_throughput"
|
||||
env:
|
||||
VLLM_USE_DEEP_GEMM: "1"
|
||||
VLLM_USE_DEEP_GEMM_MOE: "1"
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
|
||||
accuracy_threshold: 0.85
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --disable-uvicorn-access-log"
|
||||
env:
|
||||
VLLM_USE_DEEP_GEMM: "1"
|
||||
VLLM_USE_DEEP_GEMM_MOE: "1"
|
||||
VLLM_USE_DEEP_GEMM_E8M0: "0"
|
||||
@@ -0,0 +1,8 @@
|
||||
model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
|
||||
accuracy_threshold: 0.85
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"
|
||||
env:
|
||||
VLLM_USE_DEEP_GEMM: "1"
|
||||
VLLM_USE_DEEP_GEMM_MOE: "1"
|
||||
@@ -0,0 +1,8 @@
|
||||
model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"
|
||||
accuracy_threshold: 0.88
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"
|
||||
env:
|
||||
VLLM_USE_FLASHINFER_MOE_FP4: "1"
|
||||
VLLM_FLASHINFER_MOE_BACKEND: "throughput"
|
||||
@@ -0,0 +1,8 @@
|
||||
Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ht.yaml
|
||||
Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ll.yaml
|
||||
Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ht.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm.yaml
|
||||
Qwen3-30B-A3B-NvFp4-CT-fi-cutlass.yaml
|
||||
Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutlass-fi-a2av.yaml
|
||||
@@ -0,0 +1,5 @@
|
||||
model_name: "RedHatAI/Llama-4-Scout-17B-16E-Instruct-FP8-dynamic"
|
||||
accuracy_threshold: 0.92
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
|
||||
@@ -1,3 +1,4 @@
|
||||
Llama-4-Scout-Fp8-CT-vllm-cutlass.yaml
|
||||
Llama-4-Scout-Fp8-ModelOpt-fi-trtllm.yaml
|
||||
Qwen3-30B-A3B-Fp8-AutoFp8-fi-trtllm.yaml
|
||||
Qwen3-30B-A3B-NvFp4-CT-vllm-cutlass.yaml
|
||||
|
||||
@@ -5,7 +5,7 @@ Qwen3-30B-A3B-Fp8-AutoFp8-marlin.yaml
|
||||
Qwen3-30B-A3B-Fp8-AutoFp8-triton.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-marlin.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-vllm-cutlass.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-triton.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Channel-marlin.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Channel-vllm-cutlass.yaml
|
||||
Llama-4-Scout-Fp8-ModelOpt-fi-cutlass.yaml
|
||||
|
||||
@@ -61,6 +61,7 @@ def test_gsm8k_correctness(config_filename):
|
||||
server_args.extend(
|
||||
[
|
||||
"--trust-remote-code",
|
||||
"--disable-uvicorn-access-log",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -7,17 +7,22 @@ from math import prod
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
FusedMoEQuantConfig,
|
||||
fp8_w8a8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.cutlass_moe import (
|
||||
cutlass_moe_fp8,
|
||||
CutlassExpertsFp8,
|
||||
run_cutlass_moe_fp8,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts, fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
MoEPrepareAndFinalizeNoEP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
@@ -150,16 +155,15 @@ class MOETensors8Bit(MOETensors):
|
||||
|
||||
|
||||
def run_with_expert_maps(
|
||||
num_experts: int, num_local_experts: int, **cutlass_moe_kwargs
|
||||
num_experts: int,
|
||||
num_local_experts: int,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
**cutlass_moe_kwargs,
|
||||
):
|
||||
def slice_experts():
|
||||
slice_params = [
|
||||
"w1_q",
|
||||
"w2_q",
|
||||
"ab_strides1",
|
||||
"ab_strides2",
|
||||
"c_strides1",
|
||||
"c_strides2",
|
||||
"w1",
|
||||
"w2",
|
||||
]
|
||||
full_tensors = {
|
||||
k: v
|
||||
@@ -167,8 +171,6 @@ def run_with_expert_maps(
|
||||
if k in slice_params and k in cutlass_moe_kwargs
|
||||
}
|
||||
|
||||
quant_config = cutlass_moe_kwargs["quant_config"]
|
||||
|
||||
for i in range(0, num_experts, num_local_experts):
|
||||
s, e = i, i + num_local_experts
|
||||
|
||||
@@ -187,13 +189,23 @@ def run_with_expert_maps(
|
||||
new_quant_config._w1.scale = quant_config.w1_scale[s:e]
|
||||
new_quant_config._w2.scale = quant_config.w2_scale[s:e]
|
||||
|
||||
cutlass_moe_kwargs["quant_config"] = new_quant_config
|
||||
yield cutlass_moe_kwargs, new_quant_config
|
||||
|
||||
yield cutlass_moe_kwargs
|
||||
|
||||
out_tensor = torch.zeros_like(cutlass_moe_kwargs["a"])
|
||||
for kwargs in slice_experts():
|
||||
out_tensor = out_tensor + cutlass_moe_fp8(**kwargs)
|
||||
out_tensor = torch.zeros_like(cutlass_moe_kwargs["hidden_states"])
|
||||
for kwargs, new_quant_config in slice_experts():
|
||||
kernel = mk.FusedMoEModularKernel(
|
||||
MoEPrepareAndFinalizeNoEP(),
|
||||
CutlassExpertsFp8(
|
||||
out_dtype=kwargs["hidden_states"].dtype,
|
||||
# NOTE(rob): w2 is shaped as [E, hidden, intermediate]
|
||||
e=kwargs["w2"].shape[0], # type: ignore[union-attr]
|
||||
n=kwargs["w2"].shape[2], # type: ignore[union-attr]
|
||||
k=kwargs["w2"].shape[1], # type: ignore[union-attr]
|
||||
quant_config=new_quant_config,
|
||||
device="cuda",
|
||||
),
|
||||
)
|
||||
out_tensor = out_tensor + kernel(**kwargs)
|
||||
|
||||
return out_tensor
|
||||
|
||||
@@ -230,27 +242,35 @@ def run_8_bit(
|
||||
)
|
||||
|
||||
kwargs = {
|
||||
"a": moe_tensors.a,
|
||||
"w1_q": moe_tensors.w1_q, # type: ignore[union-attr]
|
||||
"w2_q": moe_tensors.w2_q, # type: ignore[union-attr]
|
||||
"hidden_states": moe_tensors.a,
|
||||
"w1": moe_tensors.w1_q, # type: ignore[union-attr]
|
||||
"w2": moe_tensors.w2_q, # type: ignore[union-attr]
|
||||
"topk_weights": topk_weights,
|
||||
"topk_ids": topk_ids,
|
||||
"ab_strides1": moe_tensors.ab_strides1,
|
||||
"ab_strides2": moe_tensors.ab_strides2,
|
||||
"c_strides1": moe_tensors.c_strides1,
|
||||
"c_strides2": moe_tensors.c_strides2,
|
||||
"quant_config": quant_config,
|
||||
}
|
||||
|
||||
num_experts = moe_tensors.w1.size(0)
|
||||
with_ep = num_local_experts is not None or num_local_experts == num_experts
|
||||
if not with_ep:
|
||||
return cutlass_moe_fp8(**kwargs)
|
||||
kernel = mk.FusedMoEModularKernel(
|
||||
MoEPrepareAndFinalizeNoEP(),
|
||||
CutlassExpertsFp8(
|
||||
out_dtype=moe_tensors.a.dtype,
|
||||
# NOTE(rob): w2 is shaped as [E, hidden, intermediate]
|
||||
e=moe_tensors.w2_q.shape[0], # type: ignore[union-attr]
|
||||
n=moe_tensors.w2_q.shape[2], # type: ignore[union-attr]
|
||||
k=moe_tensors.w2_q.shape[1], # type: ignore[union-attr]
|
||||
quant_config=quant_config,
|
||||
device="cuda",
|
||||
),
|
||||
)
|
||||
return kernel(**kwargs)
|
||||
|
||||
assert num_local_experts is not None
|
||||
return run_with_expert_maps(
|
||||
num_experts,
|
||||
num_local_experts, # type: ignore[arg-type]
|
||||
quant_config,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -11,12 +11,17 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
fp8_w8a8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
MoEPrepareAndFinalizeNoEP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
apply_flashinfer_per_tensor_scale_fp8,
|
||||
flashinfer_cutlass_moe_fp8,
|
||||
apply_fi_trtllm_fp8_per_tensor_moe,
|
||||
register_scales_for_trtllm_fp8_per_tensor_moe,
|
||||
rotate_flashinfer_fp8_moe_weights,
|
||||
rotate_weights_for_fi_trtllm_fp8_per_tensor_moe,
|
||||
swap_w13_to_w31,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import input_to_float8
|
||||
@@ -103,6 +108,7 @@ class TestData:
|
||||
w2_quantized, w2_weight_scale = quant_fp8_per_tensor_batches(w2)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.orig_dtype = torch.bfloat16
|
||||
layer.w13_weight = w13_quantized.clone()
|
||||
layer.w2_weight = w2_quantized.clone()
|
||||
layer.w13_input_scale = a1_scale
|
||||
@@ -115,10 +121,10 @@ class TestData:
|
||||
pcp_size=1,
|
||||
dp_size=1,
|
||||
ep_size=1,
|
||||
tp_rank=1,
|
||||
pcp_rank=1,
|
||||
dp_rank=1,
|
||||
ep_rank=1,
|
||||
tp_rank=0,
|
||||
pcp_rank=0,
|
||||
dp_rank=0,
|
||||
ep_rank=0,
|
||||
use_ep=False,
|
||||
all2all_backend="naive",
|
||||
)
|
||||
@@ -126,7 +132,9 @@ class TestData:
|
||||
# flashinfer expects swapped rows for w13
|
||||
layer.w13_weight.data = swap_w13_to_w31(layer.w13_weight.data)
|
||||
if is_trtllm:
|
||||
rotate_flashinfer_fp8_moe_weights(layer.w13_weight, layer.w2_weight)
|
||||
rotate_weights_for_fi_trtllm_fp8_per_tensor_moe(
|
||||
layer.w13_weight, layer.w2_weight
|
||||
)
|
||||
register_scales_for_trtllm_fp8_per_tensor_moe(
|
||||
layer,
|
||||
layer.w13_weight_scale,
|
||||
@@ -199,7 +207,7 @@ def test_flashinfer_per_tensor_moe_fp8_no_graph(
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
flashinfer_output = apply_flashinfer_per_tensor_scale_fp8(
|
||||
flashinfer_output = apply_fi_trtllm_fp8_per_tensor_moe(
|
||||
layer=td.layer,
|
||||
hidden_states=td.hidden_states,
|
||||
router_logits=score,
|
||||
@@ -277,17 +285,34 @@ def test_flashinfer_cutlass_moe_fp8_no_graph(
|
||||
td.layer.get_fused_moe_quant_config = get_fused_moe_quant_config
|
||||
td.layer.quant_method = td.layer
|
||||
|
||||
flashinfer_cutlass_output = flashinfer_cutlass_moe_fp8(
|
||||
kernel = mk.FusedMoEModularKernel(
|
||||
MoEPrepareAndFinalizeNoEP(
|
||||
defer_input_quant=quant_config.is_block_quantized
|
||||
),
|
||||
FlashInferExperts(
|
||||
out_dtype=td.layer.orig_dtype,
|
||||
quant_config=quant_config,
|
||||
ep_rank=td.layer.moe_parallel_config.ep_rank,
|
||||
ep_size=td.layer.moe_parallel_config.ep_size,
|
||||
tp_rank=td.layer.moe_parallel_config.tp_rank,
|
||||
tp_size=td.layer.moe_parallel_config.tp_size,
|
||||
use_dp=False,
|
||||
use_deepseek_fp8_block_scale=False,
|
||||
),
|
||||
)
|
||||
|
||||
flashinfer_cutlass_output = kernel(
|
||||
td.hidden_states,
|
||||
td.layer,
|
||||
td.layer.w13_weight,
|
||||
td.layer.w2_weight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
inplace=False,
|
||||
activation=activation,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=True,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
output, flashinfer_cutlass_output, atol=5.5e-2, rtol=1e-2
|
||||
)
|
||||
|
||||
@@ -15,7 +15,6 @@ from vllm.model_executor.layers.quantization.fp8 import (
|
||||
Fp8Config,
|
||||
Fp8KVCacheMethod,
|
||||
Fp8LinearMethod,
|
||||
Fp8MoeBackend,
|
||||
Fp8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
@@ -278,8 +277,18 @@ def test_scaled_fp8_quant(dtype) -> None:
|
||||
# this is the case for marlin as well as per-tensor Fp8MoEMethod
|
||||
@pytest.mark.parametrize("use_marlin", [False]) # skip True
|
||||
def test_fp8_reloading(
|
||||
method_cls, is_checkpoint_fp8_serialized, weight_block_size, use_marlin, dist_init
|
||||
method_cls,
|
||||
is_checkpoint_fp8_serialized,
|
||||
weight_block_size,
|
||||
use_marlin,
|
||||
dist_init,
|
||||
monkeypatch,
|
||||
):
|
||||
# NOTE(rob): this test fails when using DeepGEMM because the
|
||||
# shapes are invalid. Previously the test was passing because
|
||||
# we set fp8_backend to None, which sidestepped the issue.
|
||||
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0")
|
||||
|
||||
if is_checkpoint_fp8_serialized is False:
|
||||
pytest.skip("FP8 weight reloading does not support online quantization")
|
||||
|
||||
@@ -307,6 +316,7 @@ def test_fp8_reloading(
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=default_weight_loader,
|
||||
)
|
||||
method.use_marlin = use_marlin
|
||||
|
||||
else:
|
||||
layer = FusedMoE(
|
||||
@@ -325,11 +335,6 @@ def test_fp8_reloading(
|
||||
weight_loader=default_weight_loader,
|
||||
)
|
||||
|
||||
# Fp8LinearMethod uses use_marlin
|
||||
# Fp8MoEMethod uses fp8_backend
|
||||
method.use_marlin = use_marlin
|
||||
method.fp8_backend = Fp8MoeBackend.MARLIN if use_marlin else None
|
||||
|
||||
# capture weights format during loading
|
||||
original_metadata = [
|
||||
(name, param.shape, getattr(param, "weight_loader", default_weight_loader))
|
||||
|
||||
Reference in New Issue
Block a user