[MoE] Refine FlashInfer one-sided All2All integration (#51924)
Signed-off-by: Bo Li <22713281+bobboli@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
0db502c8d8
commit
c296851a7d
@@ -342,6 +342,8 @@ def _one_sided_lifecycle_worker(rank, world_size):
|
||||
top_k=2,
|
||||
num_experts=world_size * 8,
|
||||
hidden_size=4096,
|
||||
x_bytes_per_token=4096 * 2,
|
||||
x_sf_bytes_per_token=0,
|
||||
)
|
||||
|
||||
# Initialize
|
||||
@@ -409,12 +411,12 @@ def _one_sided_workspace_grow_worker(rank, world_size):
|
||||
hidden_size=4096,
|
||||
)
|
||||
nvfp4_kwargs = dict(
|
||||
dispatch_dtype_bytes_per_elem=0,
|
||||
dispatch_scale_bytes_per_token=base_kwargs["hidden_size"] // 16,
|
||||
x_bytes_per_token=base_kwargs["hidden_size"] // 2,
|
||||
x_sf_bytes_per_token=base_kwargs["hidden_size"] // 16,
|
||||
)
|
||||
bf16_kwargs = dict(
|
||||
dispatch_dtype_bytes_per_elem=2,
|
||||
dispatch_scale_bytes_per_token=0,
|
||||
x_bytes_per_token=base_kwargs["hidden_size"] * 2,
|
||||
x_sf_bytes_per_token=0,
|
||||
)
|
||||
|
||||
# First init: NVFP4-like (hidden_bytes = hidden // 2 + hidden // 16).
|
||||
@@ -780,11 +782,12 @@ def _one_sided_data_worker(rank, world_size):
|
||||
top_k=experts_per_token,
|
||||
num_experts=num_experts,
|
||||
hidden_size=hidden_size,
|
||||
# Account for the fp8 block-scale payload (a1q_scale: hidden//16 bytes
|
||||
x_bytes_per_token=hidden_size // 2,
|
||||
# Account for the fp8 block-scale payload (x_sf: hidden//16 bytes
|
||||
# per token) that is dispatched alongside the nvfp4 hidden states.
|
||||
# Without this the dispatch region is under-reserved and the combine
|
||||
# payload overflows the per-rank workspace.
|
||||
dispatch_scale_bytes_per_token=hidden_size // 16,
|
||||
x_sf_bytes_per_token=hidden_size // 16,
|
||||
)
|
||||
assert manager.initialized
|
||||
assert manager.moe_alltoall is not None
|
||||
@@ -798,17 +801,17 @@ def _one_sided_data_worker(rank, world_size):
|
||||
|
||||
# Create test data with raw tensors matching the nvfp4 payload
|
||||
# sizes the workspace was allocated for:
|
||||
# a1q: (tokens, hidden_size // 2) — nvfp4 hidden states
|
||||
# a1q_scale: (tokens, hidden_size // 16) — fp8 scaling factors
|
||||
# x: (tokens, hidden_size // 2) — nvfp4 hidden states
|
||||
# x_sf: (tokens, hidden_size // 16) — fp8 scaling factors
|
||||
torch.manual_seed(rank + 42)
|
||||
a1q = torch.randint(
|
||||
x = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(tokens_per_rank, hidden_size // 2),
|
||||
device=device,
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
a1q_scale = torch.randint(
|
||||
x_sf = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(tokens_per_rank, hidden_size // 16),
|
||||
@@ -830,15 +833,16 @@ def _one_sided_data_worker(rank, world_size):
|
||||
)
|
||||
|
||||
# --- One-sided dispatch ---
|
||||
payloads = [a1q, a1q_scale, topk_ids, topk_weights]
|
||||
payloads = [x, x_sf, topk_ids, topk_weights]
|
||||
recv_payloads = manager.moe_alltoall.dispatch(
|
||||
token_selected_experts=topk_ids,
|
||||
input_payloads=payloads,
|
||||
runtime_max_tokens_per_rank=runtime_max_tokens,
|
||||
)
|
||||
assert len(recv_payloads) == 4
|
||||
recv_a1q, recv_scale, recv_ids, recv_weights = recv_payloads
|
||||
assert recv_a1q.numel() > 0
|
||||
recv_x, recv_x_sf, recv_ids, recv_weights = recv_payloads
|
||||
assert recv_x.numel() > 0
|
||||
assert recv_x_sf.numel() > 0
|
||||
assert recv_ids.numel() > 0
|
||||
|
||||
# --- Round-trip exact verification ---
|
||||
|
||||
@@ -130,7 +130,7 @@ BACKEND_SUPPORTED_QUANTS: dict[str, set[str | None]] = {
|
||||
"mori_high_throughput": {None, "fp8", "modelopt_fp8"},
|
||||
"mori_low_latency": {None, "fp8", "modelopt_fp8"},
|
||||
"flashinfer_nvlink_two_sided": {None, "fp8_blocked", "modelopt_fp4"}, # noqa: E501
|
||||
"flashinfer_nvlink_one_sided": {None, "modelopt_fp4"}, # noqa: E501
|
||||
"flashinfer_nvlink_one_sided": {None, "fp8_blocked", "modelopt_fp4"}, # noqa: E501
|
||||
"deepep_low_latency": {None, "fp8_blocked", "modelopt_fp4"}, # noqa: E501
|
||||
"deepep_high_throughput": {None, "fp8_blocked", "modelopt_fp8", "modelopt_fp4"}, # noqa: E501
|
||||
"nixl_ep": {None, "fp8_blocked", "modelopt_fp8"},
|
||||
|
||||
Reference in New Issue
Block a user