[Qwen3.8-Flash-Next] Enable FP8 TP with FlashInfer TRTLLM MoE (#55867)
Signed-off-by: Canlin <canlinguosdu@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
parent
f1c2f6ada8
commit
08633cb5cd
@@ -0,0 +1,311 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.distributed as distributed
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
unquantized_fused_moe_method as unquantized_module,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization import fp8 as fp8_module
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
|
||||
|
||||
def _make_fp8_tp_experts(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
tp_rank,
|
||||
backend="flashinfer_trtllm",
|
||||
*,
|
||||
mock_backend=True,
|
||||
num_experts=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=640,
|
||||
quant_config=None,
|
||||
config_overrides=None,
|
||||
):
|
||||
# Exercise allocation and the public weight loader without selecting a GPU kernel.
|
||||
if mock_backend:
|
||||
monkeypatch.setattr(
|
||||
fp8_module, "select_fp8_moe_backend", lambda **kwargs: (None, None)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
unquantized_module,
|
||||
"select_unquantized_moe_backend",
|
||||
lambda **kwargs: (None, None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
fp8_module, "get_tensor_model_parallel_world_size", lambda: tp_size
|
||||
)
|
||||
# The block-shape validator imports this function from vllm.distributed.
|
||||
monkeypatch.setattr(
|
||||
distributed, "get_tensor_model_parallel_world_size", lambda: tp_size
|
||||
)
|
||||
config = make_dummy_moe_config(
|
||||
num_experts=num_experts,
|
||||
hidden_dim=hidden_dim,
|
||||
intermediate_size=intermediate_size,
|
||||
experts_per_token=min(num_experts, 10),
|
||||
)
|
||||
config.routing_method = RoutingMethodType.RenormalizeNaive
|
||||
config.moe_parallel_config.tp_size = tp_size
|
||||
config.moe_parallel_config.tp_rank = tp_rank
|
||||
config.intermediate_size_per_partition = intermediate_size // tp_size
|
||||
config.intermediate_size_per_partition_unpadded = intermediate_size // tp_size
|
||||
config.moe_backend = backend
|
||||
for field, value in (config_overrides or {}).items():
|
||||
setattr(config, field, value)
|
||||
return RoutedExperts(
|
||||
"model.layers.0.mlp.experts",
|
||||
torch.bfloat16,
|
||||
config,
|
||||
quant_config
|
||||
or Fp8Config(is_checkpoint_fp8_serialized=True, weight_block_size=[128, 128]),
|
||||
expert_map_manager=SimpleNamespace(
|
||||
local_num_experts=num_experts,
|
||||
placement_strategy="linear",
|
||||
expert_map=None,
|
||||
expert_mask=None,
|
||||
routing_tables=None,
|
||||
map_global_to_local=lambda index: index,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4, 8])
|
||||
@pytest.mark.parametrize("batched", [False, True])
|
||||
@pytest.mark.parametrize("intermediate_size", [640, 896])
|
||||
def test_fp8_block_aligned_tp_preserves_checkpoint(
|
||||
monkeypatch, tp_size, batched, intermediate_size
|
||||
):
|
||||
"""All ranks reconstruct the original dequantized projections, including
|
||||
padding-only ranks. Reloading must clear stale weights and scales.
|
||||
"""
|
||||
generator = torch.Generator().manual_seed(42)
|
||||
weights = {
|
||||
"w1": torch.randn(2, intermediate_size, 256, generator=generator).to(
|
||||
torch.float8_e4m3fn
|
||||
),
|
||||
"w3": torch.randn(2, intermediate_size, 256, generator=generator).to(
|
||||
torch.float8_e4m3fn
|
||||
),
|
||||
"w2": torch.randn(2, 256, intermediate_size, generator=generator).to(
|
||||
torch.float8_e4m3fn
|
||||
),
|
||||
}
|
||||
scales = {
|
||||
name: torch.rand(2, w.shape[1] // 128, w.shape[2] // 128, generator=generator)
|
||||
+ 0.1
|
||||
for name, w in weights.items()
|
||||
}
|
||||
reconstructed: dict[str, list[torch.Tensor]] = {name: [] for name in weights}
|
||||
num_blocks = intermediate_size // 128
|
||||
for rank in range(tp_size):
|
||||
layer = _make_fp8_tp_experts(
|
||||
monkeypatch, tp_size, rank, intermediate_size=intermediate_size
|
||||
)
|
||||
assert layer.quant_method.weight_scale_refine is None
|
||||
assert layer.quant_method.moe_block_shape == [128, 128]
|
||||
width = layer.moe_config.intermediate_size_per_partition
|
||||
assert width == ((num_blocks + tp_size - 1) // tp_size) * 128
|
||||
for name in weights:
|
||||
prefix = "w2" if name == "w2" else "w13"
|
||||
for suffix, checkpoint in (
|
||||
("weight", weights[name]),
|
||||
("weight_scale_inv", scales[name]),
|
||||
):
|
||||
param_name = f"{prefix}_{suffix}"
|
||||
param = getattr(layer, param_name)
|
||||
# Also reload after dirtying the destination, to catch padding leaks.
|
||||
for _ in range(2):
|
||||
if name != "w3":
|
||||
param.data.fill_(7)
|
||||
if batched:
|
||||
assert param.weight_loader(
|
||||
param, checkpoint, param_name, name, 0, return_success=True
|
||||
)
|
||||
else:
|
||||
for expert in range(2):
|
||||
assert param.weight_loader(
|
||||
param,
|
||||
checkpoint[expert],
|
||||
param_name,
|
||||
name,
|
||||
expert,
|
||||
return_success=True,
|
||||
)
|
||||
w = getattr(layer, f"{prefix}_weight").float()
|
||||
s = getattr(layer, f"{prefix}_weight_scale_inv")
|
||||
if name in ("w1", "w3"):
|
||||
half = 0 if name == "w1" else 1
|
||||
w = w.chunk(2, dim=1)[half]
|
||||
s = s.chunk(2, dim=1)[half]
|
||||
dim = 2 if name == "w2" else 1
|
||||
valid = max(0, min(width, intermediate_size - rank * width))
|
||||
assert torch.count_nonzero(w.narrow(dim, valid, width - valid)) == 0
|
||||
assert torch.all(s.narrow(dim, valid // 128, (width - valid) // 128) == 1)
|
||||
dequant = w * s.repeat_interleave(128, dim=1).repeat_interleave(128, dim=2)
|
||||
reconstructed[name].append(dequant.narrow(dim, 0, valid))
|
||||
for name, parts in reconstructed.items():
|
||||
expected = weights[name].float() * scales[name].repeat_interleave(
|
||||
128, dim=1
|
||||
).repeat_interleave(128, dim=2)
|
||||
assert torch.equal(torch.cat(parts, dim=2 if name == "w2" else 1), expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["auto", "triton"])
|
||||
@pytest.mark.parametrize("tp_size,block", [(2, 64), (4, 32)])
|
||||
def test_fp8_tp_default_keeps_refined_layout(monkeypatch, backend, tp_size, block):
|
||||
layer = _make_fp8_tp_experts(monkeypatch, tp_size, 0, backend)
|
||||
assert layer.moe_config.intermediate_size_per_partition == 640 // tp_size
|
||||
assert layer.quant_method.moe_block_shape == [block, block]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("hidden_dim", 192),
|
||||
("intermediate_size", 672),
|
||||
("is_lora_enabled", True),
|
||||
("has_bias", True),
|
||||
],
|
||||
)
|
||||
def test_fp8_block_aligned_tp_rejects_unsupported_layout(monkeypatch, field, value):
|
||||
with pytest.raises(ValueError, match="Block-aligned FP8 TP sharding requires"):
|
||||
_make_fp8_tp_experts(monkeypatch, 4, 0, config_overrides={field: value})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1, 5])
|
||||
def test_fp8_aligned_tp_keeps_original_layout(monkeypatch, tp_size):
|
||||
layer = _make_fp8_tp_experts(monkeypatch, tp_size, 0)
|
||||
assert layer.moe_config.intermediate_size_per_partition == 640 // tp_size
|
||||
assert not layer.moe_config.tp_shard_with_padding
|
||||
assert layer.quant_method.weight_scale_refine is None
|
||||
|
||||
|
||||
def test_fp8_skipped_layer_keeps_original_tp_layout(monkeypatch, default_vllm_config):
|
||||
layer = _make_fp8_tp_experts(
|
||||
monkeypatch,
|
||||
4,
|
||||
0,
|
||||
quant_config=Fp8Config(
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
weight_block_size=[128, 128],
|
||||
ignored_layers=["model.layers.0.mlp.experts"],
|
||||
),
|
||||
)
|
||||
assert isinstance(layer.quant_method, unquantized_module.UnquantizedFusedMoEMethod)
|
||||
assert not layer.moe_config.tp_shard_with_padding
|
||||
assert layer.moe_config.intermediate_size_per_partition == 160
|
||||
assert layer.w13_weight.shape == (2, 320, 256)
|
||||
assert layer.w2_weight.shape == (2, 256, 160)
|
||||
|
||||
|
||||
def test_fp8_block_aligned_tp_rejects_presharded_weights(monkeypatch):
|
||||
layer = _make_fp8_tp_experts(monkeypatch, 4, 0)
|
||||
with pytest.raises(ValueError, match="unsharded checkpoint"):
|
||||
layer.w13_weight.weight_loader(
|
||||
layer.w13_weight,
|
||||
torch.zeros(160, 256, dtype=torch.float8_e4m3fn),
|
||||
"w13_weight",
|
||||
"w1",
|
||||
0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4, 8])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17])
|
||||
@torch.inference_mode()
|
||||
def test_fp8_block_aligned_tp_flashinfer_matches_unsharded(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
num_tokens,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
):
|
||||
"""Compare sharded FI with unsharded FI, reporting the cross-backend
|
||||
difference separately because activation quantization can differ.
|
||||
"""
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("Requires datacenter Blackwell and FlashInfer TRTLLM")
|
||||
torch.manual_seed(42)
|
||||
e, h, n, topk = 16, 2560, 640, 10
|
||||
with torch.device("cuda"), set_current_vllm_config(VllmConfig()):
|
||||
x = torch.randn(num_tokens, h, dtype=torch.bfloat16) / 10
|
||||
logits = torch.randn(num_tokens, e, dtype=torch.float32)
|
||||
probabilities = logits.softmax(dim=-1)
|
||||
topk_weights, topk_ids = probabilities.topk(topk, dim=-1)
|
||||
topk_weights /= topk_weights.sum(dim=-1, keepdim=True)
|
||||
weights = {
|
||||
"w1": torch.randn(e, n, h).to(torch.float8_e4m3fn),
|
||||
"w3": torch.randn(e, n, h).to(torch.float8_e4m3fn),
|
||||
"w2": torch.randn(e, h, n).to(torch.float8_e4m3fn),
|
||||
}
|
||||
scales = {
|
||||
name: (torch.rand(e, w.shape[1] // 128, w.shape[2] // 128) + 0.5) / 32
|
||||
for name, w in weights.items()
|
||||
}
|
||||
ref = fused_experts(
|
||||
x,
|
||||
torch.cat([weights["w1"], weights["w3"]], dim=1),
|
||||
weights["w2"],
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
quant_config=fp8_w8a8_moe_quant_config(
|
||||
w1_scale=torch.cat([scales["w1"], scales["w3"]], dim=1),
|
||||
w2_scale=scales["w2"],
|
||||
block_shape=[128, 128],
|
||||
),
|
||||
)
|
||||
total = torch.zeros_like(ref, dtype=torch.float32)
|
||||
fi_reference = None
|
||||
for rank in range(-1, tp_size):
|
||||
layer = _make_fp8_tp_experts(
|
||||
monkeypatch,
|
||||
1 if rank == -1 else tp_size,
|
||||
max(rank, 0),
|
||||
mock_backend=False,
|
||||
num_experts=e,
|
||||
hidden_dim=h,
|
||||
)
|
||||
for name in weights:
|
||||
prefix = "w2" if name == "w2" else "w13"
|
||||
for suffix, checkpoint in (
|
||||
("weight", weights[name]),
|
||||
("weight_scale_inv", scales[name]),
|
||||
):
|
||||
param_name = f"{prefix}_{suffix}"
|
||||
param = getattr(layer, param_name)
|
||||
param.weight_loader(param, checkpoint, param_name, name, 0)
|
||||
method = layer.quant_method
|
||||
method.process_weights_after_loading(layer)
|
||||
assert method.is_monolithic
|
||||
out = method.apply_monolithic(layer, x, logits)
|
||||
assert torch.isfinite(out).all()
|
||||
if rank == -1:
|
||||
fi_reference = out.float().clone()
|
||||
else:
|
||||
total += out.float()
|
||||
# Different FP8 GEMM reduction orders need not be bitwise identical.
|
||||
assert fi_reference is not None
|
||||
relative_l2 = (total - fi_reference).norm() / fi_reference.norm().clamp_min(
|
||||
1e-8
|
||||
)
|
||||
cross_backend_l2 = (
|
||||
fi_reference - ref.float()
|
||||
).norm() / ref.float().norm().clamp_min(1e-8)
|
||||
print(
|
||||
f"TP{tp_size} M={num_tokens}: shard_l2={relative_l2.item():.6f}, "
|
||||
f"unsharded_fi_vs_triton_l2={cross_backend_l2.item():.6f}"
|
||||
)
|
||||
assert relative_l2 < 0.01, relative_l2.item()
|
||||
@@ -1309,6 +1309,8 @@ class FusedMoEConfig:
|
||||
|
||||
# Set by __post_init__
|
||||
intermediate_size_per_partition: int = -1
|
||||
# Use the allocated width as the checkpoint TP stride, including for scales.
|
||||
tp_shard_with_padding: bool = False
|
||||
rocm_aiter_fmoe_enabled: bool = False
|
||||
aiter_fmoe_shared_expert_enabled: bool = False
|
||||
|
||||
|
||||
@@ -31,11 +31,14 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
prepare_fp8_moe_layer_for_marlin,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
create_fp8_quant_key,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8Static128BlockSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -299,6 +302,80 @@ def refine_fp8_moe_block_shape(
|
||||
return [refine, refine]
|
||||
|
||||
|
||||
def pad_tp_shard_to_weight_blocks(
|
||||
config: FusedMoEConfig,
|
||||
weight_block_size: list[int],
|
||||
) -> bool:
|
||||
"""Pad the TP shard to whole checkpoint blocks, keeping scales rank-local."""
|
||||
block_n, block_k = weight_block_size
|
||||
if (
|
||||
block_n != block_k
|
||||
or config.tp_size == 1
|
||||
or config.intermediate_size_per_partition % block_n == 0
|
||||
):
|
||||
return False
|
||||
if (
|
||||
config.intermediate_size % block_n != 0
|
||||
or config.hidden_dim % block_n != 0
|
||||
or config.ep_size != 1
|
||||
or config.is_lora_enabled
|
||||
or config.has_bias
|
||||
):
|
||||
raise ValueError(
|
||||
f"Block-aligned FP8 TP sharding requires {block_n}-aligned "
|
||||
"global expert dimensions, pure TP, and no LoRA or "
|
||||
"expert bias."
|
||||
)
|
||||
config.tp_shard_with_padding = True
|
||||
config.intermediate_size_per_partition = round_up(
|
||||
config.intermediate_size_per_partition, block_n
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def resolve_fp8_moe_weight_block_shape(
|
||||
config: FusedMoEConfig,
|
||||
weight_block_size: list[int],
|
||||
activation_key: QuantKey,
|
||||
is_checkpoint_fp8_serialized: bool,
|
||||
) -> tuple[list[int], tuple[int, int] | None]:
|
||||
"""Return the TP-adapted block shape and refine factor:
|
||||
refine if kernels allow, else pad to the TP shard."""
|
||||
refined_shape = refine_fp8_moe_block_shape(config, weight_block_size)
|
||||
if is_checkpoint_fp8_serialized and config.moe_backend != "auto":
|
||||
kernel_classes = backend_to_kernel_cls(map_fp8_backend(config.moe_backend))
|
||||
can_refine = refined_shape is not None and any(
|
||||
k_cls._supports_quant_scheme(
|
||||
create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(*refined_shape)
|
||||
),
|
||||
activation_key,
|
||||
)
|
||||
for k_cls in kernel_classes
|
||||
)
|
||||
if not can_refine and pad_tp_shard_to_weight_blocks(config, weight_block_size):
|
||||
logger.info_once(
|
||||
"FP8 %s TP loading uses complete checkpoint blocks: "
|
||||
"local allocation %d, without weight requantization.",
|
||||
config.moe_backend,
|
||||
config.intermediate_size_per_partition,
|
||||
)
|
||||
return weight_block_size, None
|
||||
if refined_shape is None:
|
||||
return weight_block_size, None
|
||||
logger.info_once(
|
||||
"FP8 MoE block scales refined from %s to %s to fit "
|
||||
"the TP-sharded intermediate size %d.",
|
||||
str(weight_block_size),
|
||||
str(refined_shape),
|
||||
config.intermediate_size_per_partition,
|
||||
)
|
||||
return refined_shape, (
|
||||
weight_block_size[0] // refined_shape[0],
|
||||
weight_block_size[1] // refined_shape[1],
|
||||
)
|
||||
|
||||
|
||||
def select_fp8_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
|
||||
@@ -332,6 +332,7 @@ class RoutedExperts(PluggableLayer):
|
||||
shard_id: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
is_scale: bool = False,
|
||||
):
|
||||
"""Load grouped weight scales for group quantization or model weights
|
||||
|
||||
@@ -341,14 +342,41 @@ class RoutedExperts(PluggableLayer):
|
||||
shard_id: either w1, w2, or w3
|
||||
loaded_weight: checkpoint weight to load into the param
|
||||
tp_rank: tensor parallel rank
|
||||
is_scale: whether padding should use unit scales instead of zero weights.
|
||||
|
||||
"""
|
||||
padded_tp = self.moe_config.tp_shard_with_padding
|
||||
if padded_tp:
|
||||
destination = expert_data
|
||||
if shard_id in ("w1", "w3") and self.moe_config.is_act_and_mul:
|
||||
half = destination.shape[shard_dim] // 2
|
||||
destination = destination.narrow(
|
||||
shard_dim, 0 if shard_id == "w1" else half, half
|
||||
)
|
||||
shard_size = destination.shape[shard_dim]
|
||||
expected_size = (
|
||||
self.moe_config.intermediate_size
|
||||
* shard_size
|
||||
// self.moe_config.intermediate_size_per_partition
|
||||
)
|
||||
if loaded_weight.shape[shard_dim] != expected_size:
|
||||
raise ValueError(
|
||||
"Block-aligned TP loading expects an unsharded checkpoint "
|
||||
f"projection of size {expected_size}, got "
|
||||
f"{loaded_weight.shape[shard_dim]}."
|
||||
)
|
||||
start = min(tp_rank * shard_size, expected_size)
|
||||
size = min(shard_size, expected_size - start)
|
||||
loaded_weight = loaded_weight.narrow(shard_dim, start, size)
|
||||
destination.fill_(1 if is_scale else 0)
|
||||
|
||||
if shard_id == "w2":
|
||||
self._load_w2(
|
||||
shard_dim=shard_dim,
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
load_full=padded_tp,
|
||||
)
|
||||
elif shard_id in ("w1", "w3"):
|
||||
self._load_w13(
|
||||
@@ -357,6 +385,7 @@ class RoutedExperts(PluggableLayer):
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=tp_rank,
|
||||
load_full=padded_tp,
|
||||
)
|
||||
|
||||
def _load_per_channel_weight_scale(
|
||||
@@ -511,11 +540,12 @@ class RoutedExperts(PluggableLayer):
|
||||
shard_dim: int,
|
||||
loaded_weight: torch.Tensor,
|
||||
tp_rank: int,
|
||||
load_full: bool = False,
|
||||
):
|
||||
# Index the loaded weight for tp sharding.
|
||||
# down_proj: "RowParallel" so tp sharding on input_dim
|
||||
# Only narrow if the loaded_weight is not a scalar (0-dim tensor).
|
||||
if loaded_weight.ndim > 0:
|
||||
# Padded TP weights have already been sliced by the grouped loader.
|
||||
if not load_full and loaded_weight.ndim > 0:
|
||||
# Same padding fix as _load_w13: use unpadded per-rank size.
|
||||
tp_size = self.moe_config.moe_parallel_config.tp_size
|
||||
loaded_per_rank = loaded_weight.shape[shard_dim] // tp_size
|
||||
@@ -800,6 +830,7 @@ class RoutedExperts(PluggableLayer):
|
||||
loaded_weight=loaded_weight,
|
||||
expert_data=expert_data,
|
||||
tp_rank=self.moe_config.tp_rank,
|
||||
is_scale=True,
|
||||
)
|
||||
elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value:
|
||||
self._load_per_tensor_weight_scale(
|
||||
|
||||
@@ -8,7 +8,6 @@ import torch
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
init_fp8_linear_kernel,
|
||||
)
|
||||
@@ -24,14 +23,12 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
SharedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
refine_fp8_moe_block_shape,
|
||||
resolve_fp8_moe_weight_block_shape,
|
||||
select_fp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -90,8 +87,6 @@ if TYPE_CHECKING:
|
||||
|
||||
ACTIVATION_SCHEMES = ["static", "dynamic"]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Fp8Config(QuantizationConfig):
|
||||
"""Config class for FP8."""
|
||||
@@ -511,32 +506,16 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
# Set weight key and activation key for kernel compatibility
|
||||
if self.block_quant:
|
||||
assert self.weight_block_size is not None
|
||||
# TP shards the intermediate dim of the expert weights, so a
|
||||
# per-shard size that is not a multiple of the checkpoint's block
|
||||
# size makes the checkpoint's block scales impossible to shard
|
||||
# exactly. When a finer block size (>= 32) divides both the
|
||||
# checkpoint blocks and all involved dims, the weight scales are
|
||||
# refined to that granularity at load time (a lossless upsampling,
|
||||
# since the refined block divides the checkpoint block). The
|
||||
# refined block shape is encoded in the weight key, so the oracle
|
||||
# only selects kernels that support it (e.g. Triton, which takes
|
||||
# the block shape as a runtime argument).
|
||||
refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size)
|
||||
if refined_shape is not None:
|
||||
block_n, block_k = self.weight_block_size
|
||||
self.weight_scale_refine = (
|
||||
block_n // refined_shape[0],
|
||||
block_k // refined_shape[1],
|
||||
# Adapt the checkpoint block shape to TP sharding before
|
||||
# building the weight key.
|
||||
self.moe_block_shape, self.weight_scale_refine = (
|
||||
resolve_fp8_moe_weight_block_shape(
|
||||
self.moe,
|
||||
self.weight_block_size,
|
||||
kFp8Dynamic128Sym,
|
||||
self.quant_config.is_checkpoint_fp8_serialized,
|
||||
)
|
||||
self.moe_block_shape = refined_shape
|
||||
logger.info_once(
|
||||
"FP8 MoE block scales refined from %s to %s to fit "
|
||||
"the TP-sharded intermediate size %d.",
|
||||
str(self.weight_block_size),
|
||||
str(refined_shape),
|
||||
self.moe.intermediate_size_per_partition,
|
||||
)
|
||||
assert self.moe_block_shape is not None
|
||||
)
|
||||
weight_key = create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(*self.moe_block_shape)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user