[Qwen3.8-Flash-Next] Enable FP8 TP with FlashInfer TRTLLM MoE (#55867)

Signed-off-by: Canlin <canlinguosdu@gmail.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Canlin Guo
2026-09-17 08:56:28 +00:00
committed by GitHub
co-authored by OpenAI Codex Isotr0py
parent f1c2f6ada8
commit 08633cb5cd
5 changed files with 434 additions and 34 deletions
@@ -1309,6 +1309,8 @@ class FusedMoEConfig:
# Set by __post_init__
intermediate_size_per_partition: int = -1
# Use the allocated width as the checkpoint TP stride, including for scales.
tp_shard_with_padding: bool = False
rocm_aiter_fmoe_enabled: bool = False
aiter_fmoe_shared_expert_enabled: bool = False
@@ -31,11 +31,14 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
prepare_fp8_moe_layer_for_marlin,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
create_fp8_quant_key,
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
)
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
logger = init_logger(__name__)
@@ -299,6 +302,80 @@ def refine_fp8_moe_block_shape(
return [refine, refine]
def pad_tp_shard_to_weight_blocks(
config: FusedMoEConfig,
weight_block_size: list[int],
) -> bool:
"""Pad the TP shard to whole checkpoint blocks, keeping scales rank-local."""
block_n, block_k = weight_block_size
if (
block_n != block_k
or config.tp_size == 1
or config.intermediate_size_per_partition % block_n == 0
):
return False
if (
config.intermediate_size % block_n != 0
or config.hidden_dim % block_n != 0
or config.ep_size != 1
or config.is_lora_enabled
or config.has_bias
):
raise ValueError(
f"Block-aligned FP8 TP sharding requires {block_n}-aligned "
"global expert dimensions, pure TP, and no LoRA or "
"expert bias."
)
config.tp_shard_with_padding = True
config.intermediate_size_per_partition = round_up(
config.intermediate_size_per_partition, block_n
)
return True
def resolve_fp8_moe_weight_block_shape(
config: FusedMoEConfig,
weight_block_size: list[int],
activation_key: QuantKey,
is_checkpoint_fp8_serialized: bool,
) -> tuple[list[int], tuple[int, int] | None]:
"""Return the TP-adapted block shape and refine factor:
refine if kernels allow, else pad to the TP shard."""
refined_shape = refine_fp8_moe_block_shape(config, weight_block_size)
if is_checkpoint_fp8_serialized and config.moe_backend != "auto":
kernel_classes = backend_to_kernel_cls(map_fp8_backend(config.moe_backend))
can_refine = refined_shape is not None and any(
k_cls._supports_quant_scheme(
create_fp8_quant_key(
static=True, group_shape=GroupShape(*refined_shape)
),
activation_key,
)
for k_cls in kernel_classes
)
if not can_refine and pad_tp_shard_to_weight_blocks(config, weight_block_size):
logger.info_once(
"FP8 %s TP loading uses complete checkpoint blocks: "
"local allocation %d, without weight requantization.",
config.moe_backend,
config.intermediate_size_per_partition,
)
return weight_block_size, None
if refined_shape is None:
return weight_block_size, None
logger.info_once(
"FP8 MoE block scales refined from %s to %s to fit "
"the TP-sharded intermediate size %d.",
str(weight_block_size),
str(refined_shape),
config.intermediate_size_per_partition,
)
return refined_shape, (
weight_block_size[0] // refined_shape[0],
weight_block_size[1] // refined_shape[1],
)
def select_fp8_moe_backend(
config: FusedMoEConfig,
weight_key: QuantKey | None,
@@ -332,6 +332,7 @@ class RoutedExperts(PluggableLayer):
shard_id: str,
loaded_weight: torch.Tensor,
tp_rank: int,
is_scale: bool = False,
):
"""Load grouped weight scales for group quantization or model weights
@@ -341,14 +342,41 @@ class RoutedExperts(PluggableLayer):
shard_id: either w1, w2, or w3
loaded_weight: checkpoint weight to load into the param
tp_rank: tensor parallel rank
is_scale: whether padding should use unit scales instead of zero weights.
"""
padded_tp = self.moe_config.tp_shard_with_padding
if padded_tp:
destination = expert_data
if shard_id in ("w1", "w3") and self.moe_config.is_act_and_mul:
half = destination.shape[shard_dim] // 2
destination = destination.narrow(
shard_dim, 0 if shard_id == "w1" else half, half
)
shard_size = destination.shape[shard_dim]
expected_size = (
self.moe_config.intermediate_size
* shard_size
// self.moe_config.intermediate_size_per_partition
)
if loaded_weight.shape[shard_dim] != expected_size:
raise ValueError(
"Block-aligned TP loading expects an unsharded checkpoint "
f"projection of size {expected_size}, got "
f"{loaded_weight.shape[shard_dim]}."
)
start = min(tp_rank * shard_size, expected_size)
size = min(shard_size, expected_size - start)
loaded_weight = loaded_weight.narrow(shard_dim, start, size)
destination.fill_(1 if is_scale else 0)
if shard_id == "w2":
self._load_w2(
shard_dim=shard_dim,
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=tp_rank,
load_full=padded_tp,
)
elif shard_id in ("w1", "w3"):
self._load_w13(
@@ -357,6 +385,7 @@ class RoutedExperts(PluggableLayer):
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=tp_rank,
load_full=padded_tp,
)
def _load_per_channel_weight_scale(
@@ -511,11 +540,12 @@ class RoutedExperts(PluggableLayer):
shard_dim: int,
loaded_weight: torch.Tensor,
tp_rank: int,
load_full: bool = False,
):
# Index the loaded weight for tp sharding.
# down_proj: "RowParallel" so tp sharding on input_dim
# Only narrow if the loaded_weight is not a scalar (0-dim tensor).
if loaded_weight.ndim > 0:
# Padded TP weights have already been sliced by the grouped loader.
if not load_full and loaded_weight.ndim > 0:
# Same padding fix as _load_w13: use unpadded per-rank size.
tp_size = self.moe_config.moe_parallel_config.tp_size
loaded_per_rank = loaded_weight.shape[shard_dim] // tp_size
@@ -800,6 +830,7 @@ class RoutedExperts(PluggableLayer):
loaded_weight=loaded_weight,
expert_data=expert_data,
tp_rank=self.moe_config.tp_rank,
is_scale=True,
)
elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value:
self._load_per_tensor_weight_scale(
+11 -32
View File
@@ -8,7 +8,6 @@ import torch
import vllm.envs as envs
from vllm.config import get_current_vllm_config
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
init_fp8_linear_kernel,
)
@@ -24,14 +23,12 @@ from vllm.model_executor.layers.fused_moe import (
SharedExperts,
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
refine_fp8_moe_block_shape,
resolve_fp8_moe_weight_block_shape,
select_fp8_moe_backend,
)
from vllm.model_executor.layers.linear import (
@@ -90,8 +87,6 @@ if TYPE_CHECKING:
ACTIVATION_SCHEMES = ["static", "dynamic"]
logger = init_logger(__name__)
class Fp8Config(QuantizationConfig):
"""Config class for FP8."""
@@ -511,32 +506,16 @@ class Fp8MoEMethod(FusedMoEMethodBase):
# Set weight key and activation key for kernel compatibility
if self.block_quant:
assert self.weight_block_size is not None
# TP shards the intermediate dim of the expert weights, so a
# per-shard size that is not a multiple of the checkpoint's block
# size makes the checkpoint's block scales impossible to shard
# exactly. When a finer block size (>= 32) divides both the
# checkpoint blocks and all involved dims, the weight scales are
# refined to that granularity at load time (a lossless upsampling,
# since the refined block divides the checkpoint block). The
# refined block shape is encoded in the weight key, so the oracle
# only selects kernels that support it (e.g. Triton, which takes
# the block shape as a runtime argument).
refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size)
if refined_shape is not None:
block_n, block_k = self.weight_block_size
self.weight_scale_refine = (
block_n // refined_shape[0],
block_k // refined_shape[1],
# Adapt the checkpoint block shape to TP sharding before
# building the weight key.
self.moe_block_shape, self.weight_scale_refine = (
resolve_fp8_moe_weight_block_shape(
self.moe,
self.weight_block_size,
kFp8Dynamic128Sym,
self.quant_config.is_checkpoint_fp8_serialized,
)
self.moe_block_shape = refined_shape
logger.info_once(
"FP8 MoE block scales refined from %s to %s to fit "
"the TP-sharded intermediate size %d.",
str(self.weight_block_size),
str(refined_shape),
self.moe.intermediate_size_per_partition,
)
assert self.moe_block_shape is not None
)
weight_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(*self.moe_block_shape)
)