[Qwen3.8-Flash-Next] Enable FP8 TP with FlashInfer TRTLLM MoE (#55867)
Signed-off-by: Canlin <canlinguosdu@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
parent
f1c2f6ada8
commit
08633cb5cd
@@ -1309,6 +1309,8 @@ class FusedMoEConfig:
|
||||
|
||||
# Set by __post_init__
|
||||
intermediate_size_per_partition: int = -1
|
||||
# Use the allocated width as the checkpoint TP stride, including for scales.
|
||||
tp_shard_with_padding: bool = False
|
||||
rocm_aiter_fmoe_enabled: bool = False
|
||||
aiter_fmoe_shared_expert_enabled: bool = False
|
||||
|
||||
|
||||
@@ -31,11 +31,14 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
prepare_fp8_moe_layer_for_marlin,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
create_fp8_quant_key,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8Static128BlockSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -299,6 +302,80 @@ def refine_fp8_moe_block_shape(
|
||||
return [refine, refine]
|
||||
|
||||
|
||||
def pad_tp_shard_to_weight_blocks(
|
||||
config: FusedMoEConfig,
|
||||
weight_block_size: list[int],
|
||||
) -> bool:
|
||||
"""Pad the TP shard to whole checkpoint blocks, keeping scales rank-local."""
|
||||
block_n, block_k = weight_block_size
|
||||
if (
|
||||
block_n != block_k
|
||||
or config.tp_size == 1
|
||||
or config.intermediate_size_per_partition % block_n == 0
|
||||
):
|
||||
return False
|
||||
if (
|
||||
config.intermediate_size % block_n != 0
|
||||
or config.hidden_dim % block_n != 0
|
||||
or config.ep_size != 1
|
||||
or config.is_lora_enabled
|
||||
or config.has_bias
|
||||
):
|
||||
raise ValueError(
|
||||
f"Block-aligned FP8 TP sharding requires {block_n}-aligned "
|
||||
"global expert dimensions, pure TP, and no LoRA or "
|
||||
"expert bias."
|
||||
)
|
||||
config.tp_shard_with_padding = True
|
||||
config.intermediate_size_per_partition = round_up(
|
||||
config.intermediate_size_per_partition, block_n
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def resolve_fp8_moe_weight_block_shape(
|
||||
config: FusedMoEConfig,
|
||||
weight_block_size: list[int],
|
||||
activation_key: QuantKey,
|
||||
is_checkpoint_fp8_serialized: bool,
|
||||
) -> tuple[list[int], tuple[int, int] | None]:
|
||||
"""Return the TP-adapted block shape and refine factor:
|
||||
refine if kernels allow, else pad to the TP shard."""
|
||||
refined_shape = refine_fp8_moe_block_shape(config, weight_block_size)
|
||||
if is_checkpoint_fp8_serialized and config.moe_backend != "auto":
|
||||
kernel_classes = backend_to_kernel_cls(map_fp8_backend(config.moe_backend))
|
||||
can_refine = refined_shape is not None and any(
|
||||
k_cls._supports_quant_scheme(
|
||||
create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(*refined_shape)
|
||||
),
|
||||
activation_key,
|
||||
)
|
||||
for k_cls in kernel_classes
|
||||
)
|
||||
if not can_refine and pad_tp_shard_to_weight_blocks(config, weight_block_size):
|
||||
logger.info_once(
|
||||
"FP8 %s TP loading uses complete checkpoint blocks: "
|
||||
"local allocation %d, without weight requantization.",
|
||||
config.moe_backend,
|
||||
config.intermediate_size_per_partition,
|
||||
)
|
||||
return weight_block_size, None
|
||||
if refined_shape is None:
|
||||
return weight_block_size, None
|
||||
logger.info_once(
|
||||
"FP8 MoE block scales refined from %s to %s to fit "
|
||||
"the TP-sharded intermediate size %d.",
|
||||
str(weight_block_size),
|
||||
str(refined_shape),
|
||||
config.intermediate_size_per_partition,
|
||||
)
|
||||
return refined_shape, (
|
||||
weight_block_size[0] // refined_shape[0],
|
||||
weight_block_size[1] // refined_shape[1],
|
||||
)
|
||||
|
||||
|
||||
def select_fp8_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
|
||||
@@ -332,6 +332,7 @@ class RoutedExperts(PluggableLayer):
|
||||
shard_id: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
is_scale: bool = False,
|
||||
):
|
||||
"""Load grouped weight scales for group quantization or model weights
|
||||
|
||||
@@ -341,14 +342,41 @@ class RoutedExperts(PluggableLayer):
|
||||
shard_id: either w1, w2, or w3
|
||||
loaded_weight: checkpoint weight to load into the param
|
||||
tp_rank: tensor parallel rank
|
||||
is_scale: whether padding should use unit scales instead of zero weights.
|
||||
|
||||
"""
|
||||
padded_tp = self.moe_config.tp_shard_with_padding
|
||||
if padded_tp:
|
||||
destination = expert_data
|
||||
if shard_id in ("w1", "w3") and self.moe_config.is_act_and_mul:
|
||||
half = destination.shape[shard_dim] // 2
|
||||
destination = destination.narrow(
|
||||
shard_dim, 0 if shard_id == "w1" else half, half
|
||||
)
|
||||
shard_size = destination.shape[shard_dim]
|
||||
expected_size = (
|
||||
self.moe_config.intermediate_size
|
||||
* shard_size
|
||||
// self.moe_config.intermediate_size_per_partition
|
||||
)
|
||||
if loaded_weight.shape[shard_dim] != expected_size:
|
||||
raise ValueError(
|
||||
"Block-aligned TP loading expects an unsharded checkpoint "
|
||||
f"projection of size {expected_size}, got "
|
||||
f"{loaded_weight.shape[shard_dim]}."
|
||||
)
|
||||
start = min(tp_rank * shard_size, expected_size)
|
||||
size = min(shard_size, expected_size - start)
|
||||
loaded_weight = loaded_weight.narrow(shard_dim, start, size)
|
||||
destination.fill_(1 if is_scale else 0)
|
||||
|
||||
if shard_id == "w2":
|
||||
self._load_w2(
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
load_full=padded_tp,
|
||||
)
|
||||
elif shard_id in ("w1", "w3"):
|
||||
self._load_w13(
|
||||
@@ -357,6 +385,7 @@ class RoutedExperts(PluggableLayer):
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
load_full=padded_tp,
|
||||
)
|
||||
|
||||
def _load_per_channel_weight_scale(
|
||||
@@ -511,11 +540,12 @@ class RoutedExperts(PluggableLayer):
|
||||
shard_dim: int,
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
load_full: bool = False,
|
||||
):
|
||||
# Index the loaded weight for tp sharding.
|
||||
# down_proj: "RowParallel" so tp sharding on input_dim
|
||||
# Only narrow if the loaded_weight is not a scalar (0-dim tensor).
|
||||
if loaded_weight.ndim > 0:
|
||||
# Padded TP weights have already been sliced by the grouped loader.
|
||||
if not load_full and loaded_weight.ndim > 0:
|
||||
# Same padding fix as _load_w13: use unpadded per-rank size.
|
||||
tp_size = self.moe_config.moe_parallel_config.tp_size
|
||||
loaded_per_rank = loaded_weight.shape[shard_dim] // tp_size
|
||||
@@ -800,6 +830,7 @@ class RoutedExperts(PluggableLayer):
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=self.moe_config.tp_rank,
|
||||
is_scale=True,
|
||||
)
|
||||
elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value:
|
||||
self._load_per_tensor_weight_scale(
|
||||
|
||||
@@ -8,7 +8,6 @@ import torch
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
init_fp8_linear_kernel,
|
||||
)
|
||||
@@ -24,14 +23,12 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
SharedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
refine_fp8_moe_block_shape,
|
||||
resolve_fp8_moe_weight_block_shape,
|
||||
select_fp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -90,8 +87,6 @@ if TYPE_CHECKING:
|
||||
|
||||
ACTIVATION_SCHEMES = ["static", "dynamic"]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Fp8Config(QuantizationConfig):
|
||||
"""Config class for FP8."""
|
||||
@@ -511,32 +506,16 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
# Set weight key and activation key for kernel compatibility
|
||||
if self.block_quant:
|
||||
assert self.weight_block_size is not None
|
||||
# TP shards the intermediate dim of the expert weights, so a
|
||||
# per-shard size that is not a multiple of the checkpoint's block
|
||||
# size makes the checkpoint's block scales impossible to shard
|
||||
# exactly. When a finer block size (>= 32) divides both the
|
||||
# checkpoint blocks and all involved dims, the weight scales are
|
||||
# refined to that granularity at load time (a lossless upsampling,
|
||||
# since the refined block divides the checkpoint block). The
|
||||
# refined block shape is encoded in the weight key, so the oracle
|
||||
# only selects kernels that support it (e.g. Triton, which takes
|
||||
# the block shape as a runtime argument).
|
||||
refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size)
|
||||
if refined_shape is not None:
|
||||
block_n, block_k = self.weight_block_size
|
||||
self.weight_scale_refine = (
|
||||
block_n // refined_shape[0],
|
||||
block_k // refined_shape[1],
|
||||
# Adapt the checkpoint block shape to TP sharding before
|
||||
# building the weight key.
|
||||
self.moe_block_shape, self.weight_scale_refine = (
|
||||
resolve_fp8_moe_weight_block_shape(
|
||||
self.moe,
|
||||
self.weight_block_size,
|
||||
kFp8Dynamic128Sym,
|
||||
self.quant_config.is_checkpoint_fp8_serialized,
|
||||
)
|
||||
self.moe_block_shape = refined_shape
|
||||
logger.info_once(
|
||||
"FP8 MoE block scales refined from %s to %s to fit "
|
||||
"the TP-sharded intermediate size %d.",
|
||||
str(self.weight_block_size),
|
||||
str(refined_shape),
|
||||
self.moe.intermediate_size_per_partition,
|
||||
)
|
||||
assert self.moe_block_shape is not None
|
||||
)
|
||||
weight_key = create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(*self.moe_block_shape)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user