[Perf][MoE] Support fp8 combine in FlashInfer one-sided MoE all2all (#57995)

Signed-off-by: Ayaan Nadamal <anadamal@nvidia.com>
This commit is contained in:
Ayaan Nadamal
2026-10-02 16:29:16 -04:00
committed by GitHub
parent 111f71a6db
commit 84593955fa
3 changed files with 164 additions and 1 deletions
+129
View File
@@ -231,6 +231,8 @@ def test_one_sided_combine_into_compatibility(supports_output):
payload,
runtime_max_tokens_per_rank,
output=None,
*,
use_low_precision=False,
):
result = payload + runtime_max_tokens_per_rank
if output is None:
@@ -249,6 +251,33 @@ def test_one_sided_combine_into_compatibility(supports_output):
torch.testing.assert_close(output, payload + 2)
@pytest.mark.parametrize("low_precision_combine", [False, True])
def test_one_sided_combine_into_low_precision(low_precision_combine):
"""The fp8 combine setting is passed through to the kernel."""
from vllm.distributed.device_communicators.all2all import (
FlashInferNVLinkOneSidedManager,
)
seen_kwargs = {}
class FakeMoeAlltoAll:
def combine(self, payload, runtime_max_tokens_per_rank, output, **kwargs):
seen_kwargs.update(kwargs)
output.copy_(payload + runtime_max_tokens_per_rank)
manager = FlashInferNVLinkOneSidedManager.__new__(FlashInferNVLinkOneSidedManager)
manager.moe_alltoall = FakeMoeAlltoAll()
manager._combine_supports_output = True
manager.low_precision_combine = low_precision_combine
payload = torch.arange(4, dtype=torch.float32)
output = torch.empty_like(payload)
manager.combine_into(payload, runtime_max_tokens_per_rank=2, output=output)
assert seen_kwargs == {"use_low_precision": low_precision_combine}
torch.testing.assert_close(output, payload + 2)
# ---------------------------------------------------------------------------
# Test 1: Two-sided manager lifecycle (init, cleanup, reinit, ensure_init)
# ---------------------------------------------------------------------------
@@ -915,6 +944,106 @@ def test_one_sided_dispatch_combine(world_size):
_spawn_workers(_one_sided_data_worker, world_size, dp_size=world_size)
def _one_sided_low_precision_worker(rank, world_size):
from vllm.distributed.device_communicators.all2all import (
FlashInferNVLinkOneSidedManager,
)
from vllm.distributed.parallel_state import get_dp_group
device = torch.device(f"cuda:{rank}")
hidden_size = 256
tokens_per_rank = 32
experts_per_token = 2
num_experts = world_size * 8
manager = FlashInferNVLinkOneSidedManager(get_dp_group().cpu_group)
manager.initialize(
max_num_tokens=tokens_per_rank,
top_k=experts_per_token,
num_experts=num_experts,
hidden_size=hidden_size,
x_bytes_per_token=hidden_size // 2,
x_sf_bytes_per_token=hidden_size // 16,
)
assert manager.low_precision_combine
torch.manual_seed(rank + 42)
x = torch.randint(
0, 256, (tokens_per_rank, hidden_size // 2), device=device, dtype=torch.uint8
)
x_sf = torch.randint(
0, 256, (tokens_per_rank, hidden_size // 16), device=device, dtype=torch.uint8
)
topk_ids = torch.randint(
0,
num_experts,
(tokens_per_rank, experts_per_token),
device=device,
dtype=torch.int32,
)
topk_weights = torch.rand(
tokens_per_rank, experts_per_token, device=device, dtype=torch.float32
)
manager.moe_alltoall.dispatch(
token_selected_experts=topk_ids,
input_payloads=[x, x_sf, topk_ids, topk_weights],
runtime_max_tokens_per_rank=tokens_per_rank,
)
# Go through combine_into rather than moe_alltoall.combine: the workspace
# was sized for fp8, so the kernel must be told to send fp8. Calling the
# raw kernel here would send bf16 into fp8-sized slots and corrupt it.
expert_output = torch.ones(
world_size, tokens_per_rank, hidden_size, device=device, dtype=torch.bfloat16
)
output = torch.empty(
tokens_per_rank, hidden_size, device=device, dtype=torch.bfloat16
)
manager.combine_into(
payload=expert_output,
runtime_max_tokens_per_rank=tokens_per_rank,
output=output,
)
# Integers 1..top_k are exact in fp8_e4m3, so the fp8 transport must match
# bf16 exactly: one contribution per distinct expert-owning rank.
experts_per_rank = num_experts // world_size
expert_ranks = topk_ids // experts_per_rank
num_distinct = torch.tensor(
[len(set(row.tolist())) for row in expert_ranks],
device=device,
dtype=torch.bfloat16,
).unsqueeze(1)
torch.testing.assert_close(output, num_distinct.expand_as(output))
torch.distributed.barrier()
manager.cleanup()
@requires_multi_gpu
@requires_one_sided
@requires_ptrace
@pytest.mark.parametrize("world_size", [2])
def test_one_sided_combine_low_precision(world_size, monkeypatch):
"""Exercise the real fp8 combine transport, not a fake kernel.
A workspace/dtype mismatch corrupts the combine buffer silently rather
than raising, so this asserts the combined values are exactly right.
"""
from flashinfer.comm.trtllm_moe_alltoall import MoeAlltoAll
from vllm.utils.func_utils import supports_kw
if not supports_kw(
MoeAlltoAll.combine, "use_low_precision", allow_var_kwargs=False
):
pytest.skip("installed FlashInfer has no use_low_precision support")
monkeypatch.setenv("VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE", "1")
_spawn_workers(_one_sided_low_precision_worker, world_size, dp_size=world_size)
# ---------------------------------------------------------------------------
# Test 6: DeepEP v2 (ElasticBuffer) manager lifecycle
# ---------------------------------------------------------------------------
@@ -735,6 +735,7 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
rank: int
world_size: int
low_precision_combine: bool = False
def __init__(self, cpu_group):
assert has_flashinfer_nvlink_one_sided(), (
@@ -755,6 +756,27 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
self.top_k = 0
self.num_experts = 0
self._combine_supports_output = False
self.low_precision_combine = self._resolve_low_precision_combine()
def _resolve_low_precision_combine(self) -> bool:
"""Whether to use the low-precision combine: requested via the env var
and supported by the installed FlashInfer.
"""
if not envs.VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE:
return False
try:
supported = supports_kw(
MoeAlltoAll.combine, "use_low_precision", allow_var_kwargs=False
)
except (TypeError, ValueError):
supported = False
if not supported:
logger.warning_once(
"VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE is set, but the "
"installed FlashInfer MoeAlltoAll.combine() does not accept "
"`use_low_precision`. Falling back to a BF16 combine."
)
return supported
def initialize(
self,
@@ -772,7 +794,10 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
+ top_k * 4 # int32 topks ids
+ top_k * 4 # float32 topk weights
)
combine_payload_size_per_token = hidden_size * 2 # bf16 hidden states
# Sized from the bf16 payload passed to combine(), which is what the
# kernel checks this region against. Low-precision transport quantizes
# on write, so it does not shrink the requirement.
combine_payload_size_per_token = hidden_size * 2
needed_workspace_size = moe_a2a_get_workspace_size_per_rank(
ep_size=self.world_size,
max_num_tokens=max_num_tokens,
@@ -888,8 +913,10 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
payload=payload,
runtime_max_tokens_per_rank=runtime_max_tokens_per_rank,
output=output,
use_low_precision=self.low_precision_combine,
)
else:
# FlashInfer < 0.6.16 has neither `output` nor `use_low_precision`.
combined_output = self.moe_alltoall.combine(
payload=payload,
runtime_max_tokens_per_rank=runtime_max_tokens_per_rank,
+7
View File
@@ -222,6 +222,7 @@ if TYPE_CHECKING:
VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS: list[str] | None = None
VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto"
VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024
VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE: bool = False
VLLM_XGRAMMAR_CACHE_MB: int = 0
VLLM_REGEX_COMPILATION_TIMEOUT_S: int = 5
VLLM_MSGPACK_ZERO_COPY_THRESHOLD: int = 256
@@ -1797,6 +1798,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE": lambda: int(
os.getenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(394 * 1024 * 1024))
),
# Transmit MoE all-to-all combine (expert-output) payloads in FP8 instead
# of BF16, halving NVLink traffic on the combine leg. Only takes effect
# when the installed FlashInfer MoeAlltoAll kernel supports it.
"VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE": lambda: bool(
int(os.getenv("VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE", "0"))
),
# Control the maximum number of tokens per expert supported by the
# NVFP4 MoE CUTLASS Kernel. This value is used to create a buffer for
# the blockscale tensor of activations NVFP4 Quantization.