diff --git a/tests/kernels/moe/test_fp8_tp_loading.py b/tests/kernels/moe/test_fp8_tp_loading.py new file mode 100644 index 0000000000..680ccb812c --- /dev/null +++ b/tests/kernels/moe/test_fp8_tp_loading.py @@ -0,0 +1,311 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + +import pytest +import torch + +import vllm.distributed as distributed +from tests.kernels.moe.utils import make_dummy_moe_config +from vllm.model_executor.layers.fused_moe import ( + unquantized_fused_moe_method as unquantized_module, +) +from vllm.model_executor.layers.fused_moe.config import RoutingMethodType +from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts +from vllm.model_executor.layers.quantization import fp8 as fp8_module +from vllm.model_executor.layers.quantization.fp8 import Fp8Config + + +def _make_fp8_tp_experts( + monkeypatch, + tp_size, + tp_rank, + backend="flashinfer_trtllm", + *, + mock_backend=True, + num_experts=2, + hidden_dim=256, + intermediate_size=640, + quant_config=None, + config_overrides=None, +): + # Exercise allocation and the public weight loader without selecting a GPU kernel. + if mock_backend: + monkeypatch.setattr( + fp8_module, "select_fp8_moe_backend", lambda **kwargs: (None, None) + ) + monkeypatch.setattr( + unquantized_module, + "select_unquantized_moe_backend", + lambda **kwargs: (None, None), + ) + monkeypatch.setattr( + fp8_module, "get_tensor_model_parallel_world_size", lambda: tp_size + ) + # The block-shape validator imports this function from vllm.distributed. + monkeypatch.setattr( + distributed, "get_tensor_model_parallel_world_size", lambda: tp_size + ) + config = make_dummy_moe_config( + num_experts=num_experts, + hidden_dim=hidden_dim, + intermediate_size=intermediate_size, + experts_per_token=min(num_experts, 10), + ) + config.routing_method = RoutingMethodType.RenormalizeNaive + config.moe_parallel_config.tp_size = tp_size + config.moe_parallel_config.tp_rank = tp_rank + config.intermediate_size_per_partition = intermediate_size // tp_size + config.intermediate_size_per_partition_unpadded = intermediate_size // tp_size + config.moe_backend = backend + for field, value in (config_overrides or {}).items(): + setattr(config, field, value) + return RoutedExperts( + "model.layers.0.mlp.experts", + torch.bfloat16, + config, + quant_config + or Fp8Config(is_checkpoint_fp8_serialized=True, weight_block_size=[128, 128]), + expert_map_manager=SimpleNamespace( + local_num_experts=num_experts, + placement_strategy="linear", + expert_map=None, + expert_mask=None, + routing_tables=None, + map_global_to_local=lambda index: index, + ), + ) + + +@pytest.mark.parametrize("tp_size", [2, 4, 8]) +@pytest.mark.parametrize("batched", [False, True]) +@pytest.mark.parametrize("intermediate_size", [640, 896]) +def test_fp8_block_aligned_tp_preserves_checkpoint( + monkeypatch, tp_size, batched, intermediate_size +): + """All ranks reconstruct the original dequantized projections, including + padding-only ranks. Reloading must clear stale weights and scales. + """ + generator = torch.Generator().manual_seed(42) + weights = { + "w1": torch.randn(2, intermediate_size, 256, generator=generator).to( + torch.float8_e4m3fn + ), + "w3": torch.randn(2, intermediate_size, 256, generator=generator).to( + torch.float8_e4m3fn + ), + "w2": torch.randn(2, 256, intermediate_size, generator=generator).to( + torch.float8_e4m3fn + ), + } + scales = { + name: torch.rand(2, w.shape[1] // 128, w.shape[2] // 128, generator=generator) + + 0.1 + for name, w in weights.items() + } + reconstructed: dict[str, list[torch.Tensor]] = {name: [] for name in weights} + num_blocks = intermediate_size // 128 + for rank in range(tp_size): + layer = _make_fp8_tp_experts( + monkeypatch, tp_size, rank, intermediate_size=intermediate_size + ) + assert layer.quant_method.weight_scale_refine is None + assert layer.quant_method.moe_block_shape == [128, 128] + width = layer.moe_config.intermediate_size_per_partition + assert width == ((num_blocks + tp_size - 1) // tp_size) * 128 + for name in weights: + prefix = "w2" if name == "w2" else "w13" + for suffix, checkpoint in ( + ("weight", weights[name]), + ("weight_scale_inv", scales[name]), + ): + param_name = f"{prefix}_{suffix}" + param = getattr(layer, param_name) + # Also reload after dirtying the destination, to catch padding leaks. + for _ in range(2): + if name != "w3": + param.data.fill_(7) + if batched: + assert param.weight_loader( + param, checkpoint, param_name, name, 0, return_success=True + ) + else: + for expert in range(2): + assert param.weight_loader( + param, + checkpoint[expert], + param_name, + name, + expert, + return_success=True, + ) + w = getattr(layer, f"{prefix}_weight").float() + s = getattr(layer, f"{prefix}_weight_scale_inv") + if name in ("w1", "w3"): + half = 0 if name == "w1" else 1 + w = w.chunk(2, dim=1)[half] + s = s.chunk(2, dim=1)[half] + dim = 2 if name == "w2" else 1 + valid = max(0, min(width, intermediate_size - rank * width)) + assert torch.count_nonzero(w.narrow(dim, valid, width - valid)) == 0 + assert torch.all(s.narrow(dim, valid // 128, (width - valid) // 128) == 1) + dequant = w * s.repeat_interleave(128, dim=1).repeat_interleave(128, dim=2) + reconstructed[name].append(dequant.narrow(dim, 0, valid)) + for name, parts in reconstructed.items(): + expected = weights[name].float() * scales[name].repeat_interleave( + 128, dim=1 + ).repeat_interleave(128, dim=2) + assert torch.equal(torch.cat(parts, dim=2 if name == "w2" else 1), expected) + + +@pytest.mark.parametrize("backend", ["auto", "triton"]) +@pytest.mark.parametrize("tp_size,block", [(2, 64), (4, 32)]) +def test_fp8_tp_default_keeps_refined_layout(monkeypatch, backend, tp_size, block): + layer = _make_fp8_tp_experts(monkeypatch, tp_size, 0, backend) + assert layer.moe_config.intermediate_size_per_partition == 640 // tp_size + assert layer.quant_method.moe_block_shape == [block, block] + + +@pytest.mark.parametrize( + "field,value", + [ + ("hidden_dim", 192), + ("intermediate_size", 672), + ("is_lora_enabled", True), + ("has_bias", True), + ], +) +def test_fp8_block_aligned_tp_rejects_unsupported_layout(monkeypatch, field, value): + with pytest.raises(ValueError, match="Block-aligned FP8 TP sharding requires"): + _make_fp8_tp_experts(monkeypatch, 4, 0, config_overrides={field: value}) + + +@pytest.mark.parametrize("tp_size", [1, 5]) +def test_fp8_aligned_tp_keeps_original_layout(monkeypatch, tp_size): + layer = _make_fp8_tp_experts(monkeypatch, tp_size, 0) + assert layer.moe_config.intermediate_size_per_partition == 640 // tp_size + assert not layer.moe_config.tp_shard_with_padding + assert layer.quant_method.weight_scale_refine is None + + +def test_fp8_skipped_layer_keeps_original_tp_layout(monkeypatch, default_vllm_config): + layer = _make_fp8_tp_experts( + monkeypatch, + 4, + 0, + quant_config=Fp8Config( + is_checkpoint_fp8_serialized=True, + weight_block_size=[128, 128], + ignored_layers=["model.layers.0.mlp.experts"], + ), + ) + assert isinstance(layer.quant_method, unquantized_module.UnquantizedFusedMoEMethod) + assert not layer.moe_config.tp_shard_with_padding + assert layer.moe_config.intermediate_size_per_partition == 160 + assert layer.w13_weight.shape == (2, 320, 256) + assert layer.w2_weight.shape == (2, 256, 160) + + +def test_fp8_block_aligned_tp_rejects_presharded_weights(monkeypatch): + layer = _make_fp8_tp_experts(monkeypatch, 4, 0) + with pytest.raises(ValueError, match="unsharded checkpoint"): + layer.w13_weight.weight_loader( + layer.w13_weight, + torch.zeros(160, 256, dtype=torch.float8_e4m3fn), + "w13_weight", + "w1", + 0, + ) + + +@pytest.mark.parametrize("tp_size", [2, 4, 8]) +@pytest.mark.parametrize("num_tokens", [1, 17]) +@torch.inference_mode() +def test_fp8_block_aligned_tp_flashinfer_matches_unsharded( + monkeypatch, + tp_size, + num_tokens, + dist_init, + workspace_init, +): + """Compare sharded FI with unsharded FI, reporting the cross-backend + difference separately because activation quantization can differ. + """ + from vllm.config import VllmConfig, set_current_vllm_config + from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config + from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts + from vllm.platforms import current_platform + + if not current_platform.is_device_capability_family(100): + pytest.skip("Requires datacenter Blackwell and FlashInfer TRTLLM") + torch.manual_seed(42) + e, h, n, topk = 16, 2560, 640, 10 + with torch.device("cuda"), set_current_vllm_config(VllmConfig()): + x = torch.randn(num_tokens, h, dtype=torch.bfloat16) / 10 + logits = torch.randn(num_tokens, e, dtype=torch.float32) + probabilities = logits.softmax(dim=-1) + topk_weights, topk_ids = probabilities.topk(topk, dim=-1) + topk_weights /= topk_weights.sum(dim=-1, keepdim=True) + weights = { + "w1": torch.randn(e, n, h).to(torch.float8_e4m3fn), + "w3": torch.randn(e, n, h).to(torch.float8_e4m3fn), + "w2": torch.randn(e, h, n).to(torch.float8_e4m3fn), + } + scales = { + name: (torch.rand(e, w.shape[1] // 128, w.shape[2] // 128) + 0.5) / 32 + for name, w in weights.items() + } + ref = fused_experts( + x, + torch.cat([weights["w1"], weights["w3"]], dim=1), + weights["w2"], + topk_weights, + topk_ids, + quant_config=fp8_w8a8_moe_quant_config( + w1_scale=torch.cat([scales["w1"], scales["w3"]], dim=1), + w2_scale=scales["w2"], + block_shape=[128, 128], + ), + ) + total = torch.zeros_like(ref, dtype=torch.float32) + fi_reference = None + for rank in range(-1, tp_size): + layer = _make_fp8_tp_experts( + monkeypatch, + 1 if rank == -1 else tp_size, + max(rank, 0), + mock_backend=False, + num_experts=e, + hidden_dim=h, + ) + for name in weights: + prefix = "w2" if name == "w2" else "w13" + for suffix, checkpoint in ( + ("weight", weights[name]), + ("weight_scale_inv", scales[name]), + ): + param_name = f"{prefix}_{suffix}" + param = getattr(layer, param_name) + param.weight_loader(param, checkpoint, param_name, name, 0) + method = layer.quant_method + method.process_weights_after_loading(layer) + assert method.is_monolithic + out = method.apply_monolithic(layer, x, logits) + assert torch.isfinite(out).all() + if rank == -1: + fi_reference = out.float().clone() + else: + total += out.float() + # Different FP8 GEMM reduction orders need not be bitwise identical. + assert fi_reference is not None + relative_l2 = (total - fi_reference).norm() / fi_reference.norm().clamp_min( + 1e-8 + ) + cross_backend_l2 = ( + fi_reference - ref.float() + ).norm() / ref.float().norm().clamp_min(1e-8) + print( + f"TP{tp_size} M={num_tokens}: shard_l2={relative_l2.item():.6f}, " + f"unsharded_fi_vs_triton_l2={cross_backend_l2.item():.6f}" + ) + assert relative_l2 < 0.01, relative_l2.item() diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index 9a2f8229b1..35973e635c 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -1309,6 +1309,8 @@ class FusedMoEConfig: # Set by __post_init__ intermediate_size_per_partition: int = -1 + # Use the allocated width as the checkpoint TP stride, including for scales. + tp_shard_with_padding: bool = False rocm_aiter_fmoe_enabled: bool = False aiter_fmoe_shared_expert_enabled: bool = False diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 7c57e2e5c2..6a6ce34d16 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -31,11 +31,14 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import ( prepare_fp8_moe_layer_for_marlin, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( + GroupShape, QuantKey, + create_fp8_quant_key, kFp8Dynamic128Sym, kFp8Static128BlockSym, ) from vllm.platforms import current_platform +from vllm.utils.math_utils import round_up logger = init_logger(__name__) @@ -299,6 +302,80 @@ def refine_fp8_moe_block_shape( return [refine, refine] +def pad_tp_shard_to_weight_blocks( + config: FusedMoEConfig, + weight_block_size: list[int], +) -> bool: + """Pad the TP shard to whole checkpoint blocks, keeping scales rank-local.""" + block_n, block_k = weight_block_size + if ( + block_n != block_k + or config.tp_size == 1 + or config.intermediate_size_per_partition % block_n == 0 + ): + return False + if ( + config.intermediate_size % block_n != 0 + or config.hidden_dim % block_n != 0 + or config.ep_size != 1 + or config.is_lora_enabled + or config.has_bias + ): + raise ValueError( + f"Block-aligned FP8 TP sharding requires {block_n}-aligned " + "global expert dimensions, pure TP, and no LoRA or " + "expert bias." + ) + config.tp_shard_with_padding = True + config.intermediate_size_per_partition = round_up( + config.intermediate_size_per_partition, block_n + ) + return True + + +def resolve_fp8_moe_weight_block_shape( + config: FusedMoEConfig, + weight_block_size: list[int], + activation_key: QuantKey, + is_checkpoint_fp8_serialized: bool, +) -> tuple[list[int], tuple[int, int] | None]: + """Return the TP-adapted block shape and refine factor: + refine if kernels allow, else pad to the TP shard.""" + refined_shape = refine_fp8_moe_block_shape(config, weight_block_size) + if is_checkpoint_fp8_serialized and config.moe_backend != "auto": + kernel_classes = backend_to_kernel_cls(map_fp8_backend(config.moe_backend)) + can_refine = refined_shape is not None and any( + k_cls._supports_quant_scheme( + create_fp8_quant_key( + static=True, group_shape=GroupShape(*refined_shape) + ), + activation_key, + ) + for k_cls in kernel_classes + ) + if not can_refine and pad_tp_shard_to_weight_blocks(config, weight_block_size): + logger.info_once( + "FP8 %s TP loading uses complete checkpoint blocks: " + "local allocation %d, without weight requantization.", + config.moe_backend, + config.intermediate_size_per_partition, + ) + return weight_block_size, None + if refined_shape is None: + return weight_block_size, None + logger.info_once( + "FP8 MoE block scales refined from %s to %s to fit " + "the TP-sharded intermediate size %d.", + str(weight_block_size), + str(refined_shape), + config.intermediate_size_per_partition, + ) + return refined_shape, ( + weight_block_size[0] // refined_shape[0], + weight_block_size[1] // refined_shape[1], + ) + + def select_fp8_moe_backend( config: FusedMoEConfig, weight_key: QuantKey | None, diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index 88841070f0..42ea15dcda 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -332,6 +332,7 @@ class RoutedExperts(PluggableLayer): shard_id: str, loaded_weight: torch.Tensor, tp_rank: int, + is_scale: bool = False, ): """Load grouped weight scales for group quantization or model weights @@ -341,14 +342,41 @@ class RoutedExperts(PluggableLayer): shard_id: either w1, w2, or w3 loaded_weight: checkpoint weight to load into the param tp_rank: tensor parallel rank + is_scale: whether padding should use unit scales instead of zero weights. """ + padded_tp = self.moe_config.tp_shard_with_padding + if padded_tp: + destination = expert_data + if shard_id in ("w1", "w3") and self.moe_config.is_act_and_mul: + half = destination.shape[shard_dim] // 2 + destination = destination.narrow( + shard_dim, 0 if shard_id == "w1" else half, half + ) + shard_size = destination.shape[shard_dim] + expected_size = ( + self.moe_config.intermediate_size + * shard_size + // self.moe_config.intermediate_size_per_partition + ) + if loaded_weight.shape[shard_dim] != expected_size: + raise ValueError( + "Block-aligned TP loading expects an unsharded checkpoint " + f"projection of size {expected_size}, got " + f"{loaded_weight.shape[shard_dim]}." + ) + start = min(tp_rank * shard_size, expected_size) + size = min(shard_size, expected_size - start) + loaded_weight = loaded_weight.narrow(shard_dim, start, size) + destination.fill_(1 if is_scale else 0) + if shard_id == "w2": self._load_w2( shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, tp_rank=tp_rank, + load_full=padded_tp, ) elif shard_id in ("w1", "w3"): self._load_w13( @@ -357,6 +385,7 @@ class RoutedExperts(PluggableLayer): loaded_weight=loaded_weight, expert_data=expert_data, tp_rank=tp_rank, + load_full=padded_tp, ) def _load_per_channel_weight_scale( @@ -511,11 +540,12 @@ class RoutedExperts(PluggableLayer): shard_dim: int, loaded_weight: torch.Tensor, tp_rank: int, + load_full: bool = False, ): # Index the loaded weight for tp sharding. # down_proj: "RowParallel" so tp sharding on input_dim - # Only narrow if the loaded_weight is not a scalar (0-dim tensor). - if loaded_weight.ndim > 0: + # Padded TP weights have already been sliced by the grouped loader. + if not load_full and loaded_weight.ndim > 0: # Same padding fix as _load_w13: use unpadded per-rank size. tp_size = self.moe_config.moe_parallel_config.tp_size loaded_per_rank = loaded_weight.shape[shard_dim] // tp_size @@ -800,6 +830,7 @@ class RoutedExperts(PluggableLayer): loaded_weight=loaded_weight, expert_data=expert_data, tp_rank=self.moe_config.tp_rank, + is_scale=True, ) elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value: self._load_per_tensor_weight_scale( diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 5835af111a..c85a2c288e 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -8,7 +8,6 @@ import torch import vllm.envs as envs from vllm.config import get_current_vllm_config from vllm.distributed import get_tensor_model_parallel_world_size -from vllm.logger import init_logger from vllm.model_executor.kernels.linear import ( init_fp8_linear_kernel, ) @@ -24,14 +23,12 @@ from vllm.model_executor.layers.fused_moe import ( SharedExperts, UnquantizedFusedMoEMethod, ) -from vllm.model_executor.layers.fused_moe.config import ( - FusedMoEQuantConfig, -) +from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( convert_to_fp8_moe_kernel_format, make_fp8_moe_kernel, make_fp8_moe_quant_config, - refine_fp8_moe_block_shape, + resolve_fp8_moe_weight_block_shape, select_fp8_moe_backend, ) from vllm.model_executor.layers.linear import ( @@ -90,8 +87,6 @@ if TYPE_CHECKING: ACTIVATION_SCHEMES = ["static", "dynamic"] -logger = init_logger(__name__) - class Fp8Config(QuantizationConfig): """Config class for FP8.""" @@ -511,32 +506,16 @@ class Fp8MoEMethod(FusedMoEMethodBase): # Set weight key and activation key for kernel compatibility if self.block_quant: assert self.weight_block_size is not None - # TP shards the intermediate dim of the expert weights, so a - # per-shard size that is not a multiple of the checkpoint's block - # size makes the checkpoint's block scales impossible to shard - # exactly. When a finer block size (>= 32) divides both the - # checkpoint blocks and all involved dims, the weight scales are - # refined to that granularity at load time (a lossless upsampling, - # since the refined block divides the checkpoint block). The - # refined block shape is encoded in the weight key, so the oracle - # only selects kernels that support it (e.g. Triton, which takes - # the block shape as a runtime argument). - refined_shape = refine_fp8_moe_block_shape(self.moe, self.weight_block_size) - if refined_shape is not None: - block_n, block_k = self.weight_block_size - self.weight_scale_refine = ( - block_n // refined_shape[0], - block_k // refined_shape[1], + # Adapt the checkpoint block shape to TP sharding before + # building the weight key. + self.moe_block_shape, self.weight_scale_refine = ( + resolve_fp8_moe_weight_block_shape( + self.moe, + self.weight_block_size, + kFp8Dynamic128Sym, + self.quant_config.is_checkpoint_fp8_serialized, ) - self.moe_block_shape = refined_shape - logger.info_once( - "FP8 MoE block scales refined from %s to %s to fit " - "the TP-sharded intermediate size %d.", - str(self.weight_block_size), - str(refined_shape), - self.moe.intermediate_size_per_partition, - ) - assert self.moe_block_shape is not None + ) weight_key = create_fp8_quant_key( static=True, group_shape=GroupShape(*self.moe_block_shape) )