diff --git a/tests/distributed/test_mnnvl_alltoall.py b/tests/distributed/test_mnnvl_alltoall.py index 89fe6eb6ad..f7dac37940 100644 --- a/tests/distributed/test_mnnvl_alltoall.py +++ b/tests/distributed/test_mnnvl_alltoall.py @@ -231,6 +231,8 @@ def test_one_sided_combine_into_compatibility(supports_output): payload, runtime_max_tokens_per_rank, output=None, + *, + use_low_precision=False, ): result = payload + runtime_max_tokens_per_rank if output is None: @@ -249,6 +251,33 @@ def test_one_sided_combine_into_compatibility(supports_output): torch.testing.assert_close(output, payload + 2) +@pytest.mark.parametrize("low_precision_combine", [False, True]) +def test_one_sided_combine_into_low_precision(low_precision_combine): + """The fp8 combine setting is passed through to the kernel.""" + from vllm.distributed.device_communicators.all2all import ( + FlashInferNVLinkOneSidedManager, + ) + + seen_kwargs = {} + + class FakeMoeAlltoAll: + def combine(self, payload, runtime_max_tokens_per_rank, output, **kwargs): + seen_kwargs.update(kwargs) + output.copy_(payload + runtime_max_tokens_per_rank) + + manager = FlashInferNVLinkOneSidedManager.__new__(FlashInferNVLinkOneSidedManager) + manager.moe_alltoall = FakeMoeAlltoAll() + manager._combine_supports_output = True + manager.low_precision_combine = low_precision_combine + payload = torch.arange(4, dtype=torch.float32) + output = torch.empty_like(payload) + + manager.combine_into(payload, runtime_max_tokens_per_rank=2, output=output) + + assert seen_kwargs == {"use_low_precision": low_precision_combine} + torch.testing.assert_close(output, payload + 2) + + # --------------------------------------------------------------------------- # Test 1: Two-sided manager lifecycle (init, cleanup, reinit, ensure_init) # --------------------------------------------------------------------------- @@ -915,6 +944,106 @@ def test_one_sided_dispatch_combine(world_size): _spawn_workers(_one_sided_data_worker, world_size, dp_size=world_size) +def _one_sided_low_precision_worker(rank, world_size): + from vllm.distributed.device_communicators.all2all import ( + FlashInferNVLinkOneSidedManager, + ) + from vllm.distributed.parallel_state import get_dp_group + + device = torch.device(f"cuda:{rank}") + hidden_size = 256 + tokens_per_rank = 32 + experts_per_token = 2 + num_experts = world_size * 8 + + manager = FlashInferNVLinkOneSidedManager(get_dp_group().cpu_group) + manager.initialize( + max_num_tokens=tokens_per_rank, + top_k=experts_per_token, + num_experts=num_experts, + hidden_size=hidden_size, + x_bytes_per_token=hidden_size // 2, + x_sf_bytes_per_token=hidden_size // 16, + ) + assert manager.low_precision_combine + + torch.manual_seed(rank + 42) + x = torch.randint( + 0, 256, (tokens_per_rank, hidden_size // 2), device=device, dtype=torch.uint8 + ) + x_sf = torch.randint( + 0, 256, (tokens_per_rank, hidden_size // 16), device=device, dtype=torch.uint8 + ) + topk_ids = torch.randint( + 0, + num_experts, + (tokens_per_rank, experts_per_token), + device=device, + dtype=torch.int32, + ) + topk_weights = torch.rand( + tokens_per_rank, experts_per_token, device=device, dtype=torch.float32 + ) + + manager.moe_alltoall.dispatch( + token_selected_experts=topk_ids, + input_payloads=[x, x_sf, topk_ids, topk_weights], + runtime_max_tokens_per_rank=tokens_per_rank, + ) + + # Go through combine_into rather than moe_alltoall.combine: the workspace + # was sized for fp8, so the kernel must be told to send fp8. Calling the + # raw kernel here would send bf16 into fp8-sized slots and corrupt it. + expert_output = torch.ones( + world_size, tokens_per_rank, hidden_size, device=device, dtype=torch.bfloat16 + ) + output = torch.empty( + tokens_per_rank, hidden_size, device=device, dtype=torch.bfloat16 + ) + manager.combine_into( + payload=expert_output, + runtime_max_tokens_per_rank=tokens_per_rank, + output=output, + ) + + # Integers 1..top_k are exact in fp8_e4m3, so the fp8 transport must match + # bf16 exactly: one contribution per distinct expert-owning rank. + experts_per_rank = num_experts // world_size + expert_ranks = topk_ids // experts_per_rank + num_distinct = torch.tensor( + [len(set(row.tolist())) for row in expert_ranks], + device=device, + dtype=torch.bfloat16, + ).unsqueeze(1) + torch.testing.assert_close(output, num_distinct.expand_as(output)) + + torch.distributed.barrier() + manager.cleanup() + + +@requires_multi_gpu +@requires_one_sided +@requires_ptrace +@pytest.mark.parametrize("world_size", [2]) +def test_one_sided_combine_low_precision(world_size, monkeypatch): + """Exercise the real fp8 combine transport, not a fake kernel. + + A workspace/dtype mismatch corrupts the combine buffer silently rather + than raising, so this asserts the combined values are exactly right. + """ + from flashinfer.comm.trtllm_moe_alltoall import MoeAlltoAll + + from vllm.utils.func_utils import supports_kw + + if not supports_kw( + MoeAlltoAll.combine, "use_low_precision", allow_var_kwargs=False + ): + pytest.skip("installed FlashInfer has no use_low_precision support") + + monkeypatch.setenv("VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE", "1") + _spawn_workers(_one_sided_low_precision_worker, world_size, dp_size=world_size) + + # --------------------------------------------------------------------------- # Test 6: DeepEP v2 (ElasticBuffer) manager lifecycle # --------------------------------------------------------------------------- diff --git a/vllm/distributed/device_communicators/all2all.py b/vllm/distributed/device_communicators/all2all.py index 8658d8663b..d3c48756ca 100644 --- a/vllm/distributed/device_communicators/all2all.py +++ b/vllm/distributed/device_communicators/all2all.py @@ -735,6 +735,7 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase): rank: int world_size: int + low_precision_combine: bool = False def __init__(self, cpu_group): assert has_flashinfer_nvlink_one_sided(), ( @@ -755,6 +756,27 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase): self.top_k = 0 self.num_experts = 0 self._combine_supports_output = False + self.low_precision_combine = self._resolve_low_precision_combine() + + def _resolve_low_precision_combine(self) -> bool: + """Whether to use the low-precision combine: requested via the env var + and supported by the installed FlashInfer. + """ + if not envs.VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE: + return False + try: + supported = supports_kw( + MoeAlltoAll.combine, "use_low_precision", allow_var_kwargs=False + ) + except (TypeError, ValueError): + supported = False + if not supported: + logger.warning_once( + "VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE is set, but the " + "installed FlashInfer MoeAlltoAll.combine() does not accept " + "`use_low_precision`. Falling back to a BF16 combine." + ) + return supported def initialize( self, @@ -772,7 +794,10 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase): + top_k * 4 # int32 topks ids + top_k * 4 # float32 topk weights ) - combine_payload_size_per_token = hidden_size * 2 # bf16 hidden states + # Sized from the bf16 payload passed to combine(), which is what the + # kernel checks this region against. Low-precision transport quantizes + # on write, so it does not shrink the requirement. + combine_payload_size_per_token = hidden_size * 2 needed_workspace_size = moe_a2a_get_workspace_size_per_rank( ep_size=self.world_size, max_num_tokens=max_num_tokens, @@ -888,8 +913,10 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase): payload=payload, runtime_max_tokens_per_rank=runtime_max_tokens_per_rank, output=output, + use_low_precision=self.low_precision_combine, ) else: + # FlashInfer < 0.6.16 has neither `output` nor `use_low_precision`. combined_output = self.moe_alltoall.combine( payload=payload, runtime_max_tokens_per_rank=runtime_max_tokens_per_rank, diff --git a/vllm/envs.py b/vllm/envs.py index ac8ba21682..f51100fe8b 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -222,6 +222,7 @@ if TYPE_CHECKING: VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS: list[str] | None = None VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto" VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024 + VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE: bool = False VLLM_XGRAMMAR_CACHE_MB: int = 0 VLLM_REGEX_COMPILATION_TIMEOUT_S: int = 5 VLLM_MSGPACK_ZERO_COPY_THRESHOLD: int = 256 @@ -1797,6 +1798,12 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE": lambda: int( os.getenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(394 * 1024 * 1024)) ), + # Transmit MoE all-to-all combine (expert-output) payloads in FP8 instead + # of BF16, halving NVLink traffic on the combine leg. Only takes effect + # when the installed FlashInfer MoeAlltoAll kernel supports it. + "VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE": lambda: bool( + int(os.getenv("VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE", "0")) + ), # Control the maximum number of tokens per expert supported by the # NVFP4 MoE CUTLASS Kernel. This value is used to create a buffer for # the blockscale tensor of activations NVFP4 Quantization.