[Qwen3.8-Flash-Next] Enable FP8 TP with FlashInfer TRTLLM MoE (#55867)

Signed-off-by: Canlin <canlinguosdu@gmail.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Canlin Guo
2026-09-17 08:56:28 +00:00
committed by GitHub
co-authored by OpenAI Codex Isotr0py
parent f1c2f6ada8
commit 08633cb5cd
5 changed files with 434 additions and 34 deletions
+11 -32
View File
@@ -8,7 +8,6 @@ import torch
import vllm.envs as envs
from vllm.config import get_current_vllm_config
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
init_fp8_linear_kernel,
)
@@ -24,14 +23,12 @@ from vllm.model_executor.layers.fused_moe import (
SharedExperts,
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
refine_fp8_moe_block_shape,
resolve_fp8_moe_weight_block_shape,
select_fp8_moe_backend,
)
from vllm.model_executor.layers.linear import (
@@ -90,8 +87,6 @@ if TYPE_CHECKING:
ACTIVATION_SCHEMES = ["static", "dynamic"]
logger = init_logger(__name__)
class Fp8Config(QuantizationConfig):
"""Config class for FP8."""
@@ -511,32 +506,16 @@ class Fp8MoEMethod(FusedMoEMethodBase):
# Set weight key and activation key for kernel compatibility
if self.block_quant:
assert self.weight_block_size is not None
# TP shards the intermediate dim of the expert weights, so a
# per-shard size that is not a multiple of the checkpoint's block
# size makes the checkpoint's block scales impossible to shard
# exactly. When a finer block size (>= 32) divides both the
# checkpoint blocks and all involved dims, the weight scales are
# refined to that granularity at load time (a lossless upsampling,
# since the refined block divides the checkpoint block). The
# refined block shape is encoded in the weight key, so the oracle
# only selects kernels that support it (e.g. Triton, which takes
# the block shape as a runtime argument).
refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size)
if refined_shape is not None:
block_n, block_k = self.weight_block_size
self.weight_scale_refine = (
block_n // refined_shape[0],
block_k // refined_shape[1],
# Adapt the checkpoint block shape to TP sharding before
# building the weight key.
self.moe_block_shape, self.weight_scale_refine = (
resolve_fp8_moe_weight_block_shape(
self.moe,
self.weight_block_size,
kFp8Dynamic128Sym,
self.quant_config.is_checkpoint_fp8_serialized,
)
self.moe_block_shape = refined_shape
logger.info_once(
"FP8 MoE block scales refined from %s to %s to fit "
"the TP-sharded intermediate size %d.",
str(self.weight_block_size),
str(refined_shape),
self.moe.intermediate_size_per_partition,
)
assert self.moe_block_shape is not None
)
weight_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(*self.moe_block_shape)
)