[Qwen3.8-Flash-Next] Enable FP8 TP with FlashInfer TRTLLM MoE (#55867)
Signed-off-by: Canlin <canlinguosdu@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
parent
f1c2f6ada8
commit
08633cb5cd
@@ -8,7 +8,6 @@ import torch
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
init_fp8_linear_kernel,
|
||||
)
|
||||
@@ -24,14 +23,12 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
SharedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
refine_fp8_moe_block_shape,
|
||||
resolve_fp8_moe_weight_block_shape,
|
||||
select_fp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -90,8 +87,6 @@ if TYPE_CHECKING:
|
||||
|
||||
ACTIVATION_SCHEMES = ["static", "dynamic"]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Fp8Config(QuantizationConfig):
|
||||
"""Config class for FP8."""
|
||||
@@ -511,32 +506,16 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
# Set weight key and activation key for kernel compatibility
|
||||
if self.block_quant:
|
||||
assert self.weight_block_size is not None
|
||||
# TP shards the intermediate dim of the expert weights, so a
|
||||
# per-shard size that is not a multiple of the checkpoint's block
|
||||
# size makes the checkpoint's block scales impossible to shard
|
||||
# exactly. When a finer block size (>= 32) divides both the
|
||||
# checkpoint blocks and all involved dims, the weight scales are
|
||||
# refined to that granularity at load time (a lossless upsampling,
|
||||
# since the refined block divides the checkpoint block). The
|
||||
# refined block shape is encoded in the weight key, so the oracle
|
||||
# only selects kernels that support it (e.g. Triton, which takes
|
||||
# the block shape as a runtime argument).
|
||||
refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size)
|
||||
if refined_shape is not None:
|
||||
block_n, block_k = self.weight_block_size
|
||||
self.weight_scale_refine = (
|
||||
block_n // refined_shape[0],
|
||||
block_k // refined_shape[1],
|
||||
# Adapt the checkpoint block shape to TP sharding before
|
||||
# building the weight key.
|
||||
self.moe_block_shape, self.weight_scale_refine = (
|
||||
resolve_fp8_moe_weight_block_shape(
|
||||
self.moe,
|
||||
self.weight_block_size,
|
||||
kFp8Dynamic128Sym,
|
||||
self.quant_config.is_checkpoint_fp8_serialized,
|
||||
)
|
||||
self.moe_block_shape = refined_shape
|
||||
logger.info_once(
|
||||
"FP8 MoE block scales refined from %s to %s to fit "
|
||||
"the TP-sharded intermediate size %d.",
|
||||
str(self.weight_block_size),
|
||||
str(refined_shape),
|
||||
self.moe.intermediate_size_per_partition,
|
||||
)
|
||||
assert self.moe_block_shape is not None
|
||||
)
|
||||
weight_key = create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(*self.moe_block_shape)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user