Fix Flashinfer CUTLASS MOE Allgather (#21963)
Signed-off-by: Shu Wang <shuw@nvidia.com>
This commit is contained in:
@@ -4,7 +4,6 @@ from typing import Any, Optional
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.distributed import get_dp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
@@ -14,20 +13,8 @@ from vllm.model_executor.layers.fused_moe.utils import (
|
||||
from vllm.utils.flashinfer import nvfp4_block_scale_interleave
|
||||
|
||||
|
||||
def get_local_sizes(local_tokens):
|
||||
cu_sizes = get_forward_context().dp_metadata.cu_tokens_across_dp_cpu
|
||||
sizes = [cu_sizes[0].item()]
|
||||
for i in range(1, len(cu_sizes)):
|
||||
sizes.append((cu_sizes[i] - cu_sizes[i - 1]).item())
|
||||
max_num_tokens = envs.VLLM_MOE_DP_CHUNK_SIZE
|
||||
sizes_chunked = [max_num_tokens] * len(sizes)
|
||||
if local_tokens < max_num_tokens:
|
||||
# When the number of local tokens is less than max_num_tokens, all other
|
||||
# ranks will also have fewer than max_num_tokens. The remaining tokens
|
||||
# are accounted for as residual.
|
||||
sizes_chunked = [x % max_num_tokens for x in sizes]
|
||||
|
||||
return sizes_chunked
|
||||
def get_local_sizes():
|
||||
return get_forward_context().dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
|
||||
|
||||
class FlashInferCutlassMoEPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize):
|
||||
@@ -90,7 +77,7 @@ class FlashInferCutlassMoEPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize):
|
||||
topk_weights, topk_ids, a1q, a1q_scale = \
|
||||
get_dp_group().all_gatherv([topk_weights, topk_ids, a1q, a1q_scale], # noqa: E501
|
||||
dim=0,
|
||||
sizes=get_local_sizes(local_tokens))
|
||||
sizes=get_local_sizes())
|
||||
a1_m, a1_n = a1q.shape
|
||||
a1q_scale = nvfp4_block_scale_interleave(a1q_scale)
|
||||
|
||||
@@ -107,8 +94,5 @@ class FlashInferCutlassMoEPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize):
|
||||
['use_dp', 'local_tokens'])
|
||||
if use_dp:
|
||||
fused_expert_output = get_dp_group().reduce_scatterv(
|
||||
fused_expert_output,
|
||||
dim=0,
|
||||
sizes=get_local_sizes(local_tokens),
|
||||
)
|
||||
fused_expert_output, dim=0, sizes=get_local_sizes())
|
||||
output.copy_(fused_expert_output)
|
||||
|
||||
@@ -1570,18 +1570,19 @@ class FusedMoE(torch.nn.Module):
|
||||
max_tokens_across_dp = ctx.dp_metadata.max_tokens_across_dp_cpu
|
||||
moe_dp_chunk_size_per_rank = self.moe_config.max_num_tokens
|
||||
num_tokens = full_hidden_states.size(0)
|
||||
for chunk_start_ in range(0, max_tokens_across_dp,
|
||||
moe_dp_chunk_size_per_rank):
|
||||
for chunk_idx, chunk_start_ in enumerate(
|
||||
range(0, max_tokens_across_dp, moe_dp_chunk_size_per_rank)):
|
||||
chunk_start = chunk_start_
|
||||
chunk_end = min(chunk_start + moe_dp_chunk_size_per_rank,
|
||||
max_tokens_across_dp)
|
||||
# clamp start and end
|
||||
chunk_start = min(chunk_start, num_tokens - 1)
|
||||
chunk_end = min(chunk_end, num_tokens)
|
||||
|
||||
process_chunk(chunk_start,
|
||||
chunk_end,
|
||||
skip_result_store=chunk_start_ >= num_tokens)
|
||||
with ctx.dp_metadata.chunked_sizes(moe_dp_chunk_size_per_rank,
|
||||
chunk_idx):
|
||||
process_chunk(chunk_start,
|
||||
chunk_end,
|
||||
skip_result_store=chunk_start_ >= num_tokens)
|
||||
|
||||
return full_final_hidden_states
|
||||
|
||||
|
||||
Reference in New Issue
Block a user