[Perf][MoE] Support fp8 combine in FlashInfer one-sided MoE all2all (#57995)
Signed-off-by: Ayaan Nadamal <anadamal@nvidia.com>
This commit is contained in:
@@ -231,6 +231,8 @@ def test_one_sided_combine_into_compatibility(supports_output):
|
||||
payload,
|
||||
runtime_max_tokens_per_rank,
|
||||
output=None,
|
||||
*,
|
||||
use_low_precision=False,
|
||||
):
|
||||
result = payload + runtime_max_tokens_per_rank
|
||||
if output is None:
|
||||
@@ -249,6 +251,33 @@ def test_one_sided_combine_into_compatibility(supports_output):
|
||||
torch.testing.assert_close(output, payload + 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("low_precision_combine", [False, True])
|
||||
def test_one_sided_combine_into_low_precision(low_precision_combine):
|
||||
"""The fp8 combine setting is passed through to the kernel."""
|
||||
from vllm.distributed.device_communicators.all2all import (
|
||||
FlashInferNVLinkOneSidedManager,
|
||||
)
|
||||
|
||||
seen_kwargs = {}
|
||||
|
||||
class FakeMoeAlltoAll:
|
||||
def combine(self, payload, runtime_max_tokens_per_rank, output, **kwargs):
|
||||
seen_kwargs.update(kwargs)
|
||||
output.copy_(payload + runtime_max_tokens_per_rank)
|
||||
|
||||
manager = FlashInferNVLinkOneSidedManager.__new__(FlashInferNVLinkOneSidedManager)
|
||||
manager.moe_alltoall = FakeMoeAlltoAll()
|
||||
manager._combine_supports_output = True
|
||||
manager.low_precision_combine = low_precision_combine
|
||||
payload = torch.arange(4, dtype=torch.float32)
|
||||
output = torch.empty_like(payload)
|
||||
|
||||
manager.combine_into(payload, runtime_max_tokens_per_rank=2, output=output)
|
||||
|
||||
assert seen_kwargs == {"use_low_precision": low_precision_combine}
|
||||
torch.testing.assert_close(output, payload + 2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 1: Two-sided manager lifecycle (init, cleanup, reinit, ensure_init)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -915,6 +944,106 @@ def test_one_sided_dispatch_combine(world_size):
|
||||
_spawn_workers(_one_sided_data_worker, world_size, dp_size=world_size)
|
||||
|
||||
|
||||
def _one_sided_low_precision_worker(rank, world_size):
|
||||
from vllm.distributed.device_communicators.all2all import (
|
||||
FlashInferNVLinkOneSidedManager,
|
||||
)
|
||||
from vllm.distributed.parallel_state import get_dp_group
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
hidden_size = 256
|
||||
tokens_per_rank = 32
|
||||
experts_per_token = 2
|
||||
num_experts = world_size * 8
|
||||
|
||||
manager = FlashInferNVLinkOneSidedManager(get_dp_group().cpu_group)
|
||||
manager.initialize(
|
||||
max_num_tokens=tokens_per_rank,
|
||||
top_k=experts_per_token,
|
||||
num_experts=num_experts,
|
||||
hidden_size=hidden_size,
|
||||
x_bytes_per_token=hidden_size // 2,
|
||||
x_sf_bytes_per_token=hidden_size // 16,
|
||||
)
|
||||
assert manager.low_precision_combine
|
||||
|
||||
torch.manual_seed(rank + 42)
|
||||
x = torch.randint(
|
||||
0, 256, (tokens_per_rank, hidden_size // 2), device=device, dtype=torch.uint8
|
||||
)
|
||||
x_sf = torch.randint(
|
||||
0, 256, (tokens_per_rank, hidden_size // 16), device=device, dtype=torch.uint8
|
||||
)
|
||||
topk_ids = torch.randint(
|
||||
0,
|
||||
num_experts,
|
||||
(tokens_per_rank, experts_per_token),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
topk_weights = torch.rand(
|
||||
tokens_per_rank, experts_per_token, device=device, dtype=torch.float32
|
||||
)
|
||||
|
||||
manager.moe_alltoall.dispatch(
|
||||
token_selected_experts=topk_ids,
|
||||
input_payloads=[x, x_sf, topk_ids, topk_weights],
|
||||
runtime_max_tokens_per_rank=tokens_per_rank,
|
||||
)
|
||||
|
||||
# Go through combine_into rather than moe_alltoall.combine: the workspace
|
||||
# was sized for fp8, so the kernel must be told to send fp8. Calling the
|
||||
# raw kernel here would send bf16 into fp8-sized slots and corrupt it.
|
||||
expert_output = torch.ones(
|
||||
world_size, tokens_per_rank, hidden_size, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
output = torch.empty(
|
||||
tokens_per_rank, hidden_size, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
manager.combine_into(
|
||||
payload=expert_output,
|
||||
runtime_max_tokens_per_rank=tokens_per_rank,
|
||||
output=output,
|
||||
)
|
||||
|
||||
# Integers 1..top_k are exact in fp8_e4m3, so the fp8 transport must match
|
||||
# bf16 exactly: one contribution per distinct expert-owning rank.
|
||||
experts_per_rank = num_experts // world_size
|
||||
expert_ranks = topk_ids // experts_per_rank
|
||||
num_distinct = torch.tensor(
|
||||
[len(set(row.tolist())) for row in expert_ranks],
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
).unsqueeze(1)
|
||||
torch.testing.assert_close(output, num_distinct.expand_as(output))
|
||||
|
||||
torch.distributed.barrier()
|
||||
manager.cleanup()
|
||||
|
||||
|
||||
@requires_multi_gpu
|
||||
@requires_one_sided
|
||||
@requires_ptrace
|
||||
@pytest.mark.parametrize("world_size", [2])
|
||||
def test_one_sided_combine_low_precision(world_size, monkeypatch):
|
||||
"""Exercise the real fp8 combine transport, not a fake kernel.
|
||||
|
||||
A workspace/dtype mismatch corrupts the combine buffer silently rather
|
||||
than raising, so this asserts the combined values are exactly right.
|
||||
"""
|
||||
from flashinfer.comm.trtllm_moe_alltoall import MoeAlltoAll
|
||||
|
||||
from vllm.utils.func_utils import supports_kw
|
||||
|
||||
if not supports_kw(
|
||||
MoeAlltoAll.combine, "use_low_precision", allow_var_kwargs=False
|
||||
):
|
||||
pytest.skip("installed FlashInfer has no use_low_precision support")
|
||||
|
||||
monkeypatch.setenv("VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE", "1")
|
||||
_spawn_workers(_one_sided_low_precision_worker, world_size, dp_size=world_size)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test 6: DeepEP v2 (ElasticBuffer) manager lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -735,6 +735,7 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
|
||||
|
||||
rank: int
|
||||
world_size: int
|
||||
low_precision_combine: bool = False
|
||||
|
||||
def __init__(self, cpu_group):
|
||||
assert has_flashinfer_nvlink_one_sided(), (
|
||||
@@ -755,6 +756,27 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
|
||||
self.top_k = 0
|
||||
self.num_experts = 0
|
||||
self._combine_supports_output = False
|
||||
self.low_precision_combine = self._resolve_low_precision_combine()
|
||||
|
||||
def _resolve_low_precision_combine(self) -> bool:
|
||||
"""Whether to use the low-precision combine: requested via the env var
|
||||
and supported by the installed FlashInfer.
|
||||
"""
|
||||
if not envs.VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE:
|
||||
return False
|
||||
try:
|
||||
supported = supports_kw(
|
||||
MoeAlltoAll.combine, "use_low_precision", allow_var_kwargs=False
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
supported = False
|
||||
if not supported:
|
||||
logger.warning_once(
|
||||
"VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE is set, but the "
|
||||
"installed FlashInfer MoeAlltoAll.combine() does not accept "
|
||||
"`use_low_precision`. Falling back to a BF16 combine."
|
||||
)
|
||||
return supported
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
@@ -772,7 +794,10 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
|
||||
+ top_k * 4 # int32 topks ids
|
||||
+ top_k * 4 # float32 topk weights
|
||||
)
|
||||
combine_payload_size_per_token = hidden_size * 2 # bf16 hidden states
|
||||
# Sized from the bf16 payload passed to combine(), which is what the
|
||||
# kernel checks this region against. Low-precision transport quantizes
|
||||
# on write, so it does not shrink the requirement.
|
||||
combine_payload_size_per_token = hidden_size * 2
|
||||
needed_workspace_size = moe_a2a_get_workspace_size_per_rank(
|
||||
ep_size=self.world_size,
|
||||
max_num_tokens=max_num_tokens,
|
||||
@@ -888,8 +913,10 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
|
||||
payload=payload,
|
||||
runtime_max_tokens_per_rank=runtime_max_tokens_per_rank,
|
||||
output=output,
|
||||
use_low_precision=self.low_precision_combine,
|
||||
)
|
||||
else:
|
||||
# FlashInfer < 0.6.16 has neither `output` nor `use_low_precision`.
|
||||
combined_output = self.moe_alltoall.combine(
|
||||
payload=payload,
|
||||
runtime_max_tokens_per_rank=runtime_max_tokens_per_rank,
|
||||
|
||||
@@ -222,6 +222,7 @@ if TYPE_CHECKING:
|
||||
VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS: list[str] | None = None
|
||||
VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto"
|
||||
VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024
|
||||
VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE: bool = False
|
||||
VLLM_XGRAMMAR_CACHE_MB: int = 0
|
||||
VLLM_REGEX_COMPILATION_TIMEOUT_S: int = 5
|
||||
VLLM_MSGPACK_ZERO_COPY_THRESHOLD: int = 256
|
||||
@@ -1797,6 +1798,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE": lambda: int(
|
||||
os.getenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", str(394 * 1024 * 1024))
|
||||
),
|
||||
# Transmit MoE all-to-all combine (expert-output) payloads in FP8 instead
|
||||
# of BF16, halving NVLink traffic on the combine leg. Only takes effect
|
||||
# when the installed FlashInfer MoeAlltoAll kernel supports it.
|
||||
"VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE": lambda: bool(
|
||||
int(os.getenv("VLLM_FLASHINFER_MOE_A2A_LOW_PRECISION_COMBINE", "0"))
|
||||
),
|
||||
# Control the maximum number of tokens per expert supported by the
|
||||
# NVFP4 MoE CUTLASS Kernel. This value is used to create a buffer for
|
||||
# the blockscale tensor of activations NVFP4 Quantization.
|
||||
|
||||
Reference in New Issue
Block a user