[ROCm][FEAT] Fuse DeepSeek shared experts into AITER fused_moe ops (#24097)
Signed-off-by: chenjun <junchen2@amd.com> Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com> Co-authored-by: valarLip <103567126+valarLip@users.noreply.github.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -5,6 +5,7 @@ from abc import abstractmethod
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import nullcontext
|
||||
from enum import Enum
|
||||
from functools import partial
|
||||
from typing import Literal, get_args, overload
|
||||
|
||||
import torch
|
||||
@@ -12,7 +13,7 @@ import torch.nn.functional as F
|
||||
from torch.nn.parameter import UninitializedParameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.parallel import ExpertPlacementStrategy
|
||||
from vllm.distributed import (
|
||||
get_dp_group,
|
||||
@@ -39,6 +40,8 @@ from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import (
|
||||
init_aiter_topK_meta_data,
|
||||
is_rocm_aiter_fusion_shared_expert_enabled,
|
||||
is_rocm_aiter_moe_enabled,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routing_simulator import RoutingSimulator
|
||||
@@ -87,7 +90,7 @@ else:
|
||||
|
||||
if is_rocm_aiter_moe_enabled():
|
||||
from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( # noqa: E501
|
||||
rocm_aiter_grouped_topk as grouped_topk,
|
||||
rocm_aiter_grouped_topk as grouped_topk_aiter,
|
||||
)
|
||||
else:
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import grouped_topk
|
||||
@@ -634,6 +637,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
global_num_experts=global_num_experts,
|
||||
zero_expert_num=zero_expert_num,
|
||||
zero_expert_type=zero_expert_type,
|
||||
num_fused_shared_experts=layer.num_fused_shared_experts,
|
||||
)
|
||||
|
||||
if self.rocm_aiter_moe_enabled:
|
||||
@@ -860,7 +864,8 @@ def determine_expert_map(
|
||||
ep_rank: int,
|
||||
global_num_experts: int,
|
||||
expert_placement_strategy: ExpertPlacementStrategy = "linear",
|
||||
) -> tuple[int, torch.Tensor | None]:
|
||||
num_fused_shared_experts: int = 0,
|
||||
) -> tuple[int, torch.Tensor | None, torch.Tensor | None]:
|
||||
"""
|
||||
Calculates how many experts should be assigned to each rank for EP and
|
||||
creates a mapping from global to local expert index. Experts are
|
||||
@@ -882,10 +887,16 @@ def determine_expert_map(
|
||||
(global_num_experts,) mapping from global to local index.
|
||||
Contains -1 for experts not assigned to the current rank.
|
||||
Returns None if ep_size is 1.
|
||||
- expert_mask (Optional[torch.Tensor]): A tensor of shape
|
||||
(global_num_experts + num_fused_shared_experts + 1,)
|
||||
containing 1 for experts assigned to the current rank
|
||||
and 0 for sentinel.
|
||||
Returns None if ep_size is 1.
|
||||
Used only when AITER MOE is enabled.
|
||||
"""
|
||||
assert ep_size > 0
|
||||
if ep_size == 1:
|
||||
return (global_num_experts, None)
|
||||
return (global_num_experts, None, None)
|
||||
|
||||
# Distribute experts as evenly as possible to each rank.
|
||||
base_experts = global_num_experts // ep_size
|
||||
@@ -914,7 +925,26 @@ def determine_expert_map(
|
||||
f"'{expert_placement_strategy}', expected one of "
|
||||
f"{get_args(ExpertPlacementStrategy)}"
|
||||
)
|
||||
return (local_num_experts, expert_map)
|
||||
|
||||
expert_mask = None
|
||||
if is_rocm_aiter_moe_enabled():
|
||||
expert_mask = torch.ones(
|
||||
(global_num_experts + num_fused_shared_experts + 1,), dtype=torch.int32
|
||||
)
|
||||
expert_mask[-1] = 0
|
||||
expert_mask[:global_num_experts] = expert_map > -1
|
||||
expert_map = torch.cat(
|
||||
(
|
||||
expert_map,
|
||||
torch.tensor(
|
||||
[local_num_experts + i for i in range(num_fused_shared_experts)],
|
||||
dtype=torch.int32,
|
||||
),
|
||||
),
|
||||
dim=0,
|
||||
)
|
||||
|
||||
return (local_num_experts, expert_map, expert_mask)
|
||||
|
||||
|
||||
def get_compressed_expert_map(expert_map: torch.Tensor) -> str:
|
||||
@@ -1040,6 +1070,7 @@ class FusedMoE(CustomOp):
|
||||
zero_expert_num: int | None = 0,
|
||||
zero_expert_type: str | None = None,
|
||||
expert_mapping: list[tuple[str, str, int, str]] | None = None,
|
||||
n_shared_experts: int | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
if params_dtype is None:
|
||||
@@ -1096,6 +1127,22 @@ class FusedMoE(CustomOp):
|
||||
self.logical_to_physical_map: torch.Tensor | None = None
|
||||
self.logical_replica_count: torch.Tensor | None = None
|
||||
|
||||
# ROCm aiter shared experts fusion
|
||||
self.num_fused_shared_experts = (
|
||||
n_shared_experts
|
||||
if n_shared_experts is not None
|
||||
and is_rocm_aiter_fusion_shared_expert_enabled()
|
||||
else 0
|
||||
)
|
||||
if (
|
||||
not is_rocm_aiter_fusion_shared_expert_enabled()
|
||||
and self.num_fused_shared_experts != 0
|
||||
):
|
||||
raise ValueError(
|
||||
"n_shared_experts is only supported on ROCm aiter when "
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled"
|
||||
)
|
||||
|
||||
# Determine expert maps
|
||||
if self.use_ep:
|
||||
if self.enable_eplb:
|
||||
@@ -1129,14 +1176,16 @@ class FusedMoE(CustomOp):
|
||||
expert_placement_strategy = "linear"
|
||||
|
||||
self.expert_map: torch.Tensor | None
|
||||
local_num_experts, expert_map = determine_expert_map(
|
||||
local_num_experts, expert_map, expert_mask = determine_expert_map(
|
||||
ep_size=self.ep_size,
|
||||
ep_rank=self.ep_rank,
|
||||
global_num_experts=self.global_num_experts,
|
||||
expert_placement_strategy=expert_placement_strategy,
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
)
|
||||
self.local_num_experts = local_num_experts
|
||||
self.register_buffer("expert_map", expert_map)
|
||||
self.register_buffer("expert_mask", expert_mask)
|
||||
logger.info_once(
|
||||
"[EP Rank %s/%s] Expert parallelism is enabled. Expert "
|
||||
"placement strategy: %s. Local/global"
|
||||
@@ -1150,10 +1199,18 @@ class FusedMoE(CustomOp):
|
||||
get_compressed_expert_map(self.expert_map),
|
||||
)
|
||||
else:
|
||||
self.local_num_experts, self.expert_map = (self.global_num_experts, None)
|
||||
self.local_num_experts, self.expert_map, self.expert_mask = (
|
||||
self.global_num_experts,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
self.top_k = top_k
|
||||
|
||||
self._init_aiter_shared_experts_topK_buffer(
|
||||
vllm_config=vllm_config, dp_size=dp_size_
|
||||
)
|
||||
|
||||
assert intermediate_size % self.tp_size == 0
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size_per_partition = intermediate_size // self.tp_size
|
||||
@@ -1327,13 +1384,18 @@ class FusedMoE(CustomOp):
|
||||
# ep_size and ep_rank should already be updated
|
||||
assert self.expert_map is not None
|
||||
with self.expert_map.device:
|
||||
local_num_experts, expert_map = determine_expert_map(
|
||||
local_num_experts, expert_map, expert_mask = determine_expert_map(
|
||||
ep_size=self.ep_size,
|
||||
ep_rank=self.ep_rank,
|
||||
global_num_experts=self.global_num_experts,
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
)
|
||||
self.local_num_experts = local_num_experts
|
||||
self.register_buffer("expert_map", expert_map)
|
||||
self.register_buffer("expert_mask", expert_mask)
|
||||
self._init_aiter_shared_experts_topK_buffer(
|
||||
vllm_config=get_current_vllm_config(), dp_size=get_dp_group().world_size
|
||||
)
|
||||
|
||||
def _load_per_tensor_weight_scale(
|
||||
self,
|
||||
@@ -1504,6 +1566,24 @@ class FusedMoE(CustomOp):
|
||||
return expert_id
|
||||
return self.expert_map[expert_id].item()
|
||||
|
||||
def _init_aiter_shared_experts_topK_buffer(
|
||||
self, vllm_config: VllmConfig, dp_size: int
|
||||
):
|
||||
if is_rocm_aiter_fusion_shared_expert_enabled():
|
||||
if self.num_fused_shared_experts > 0:
|
||||
init_aiter_topK_meta_data(
|
||||
n_routed_experts=self.global_num_experts,
|
||||
n_shared_experts=self.num_fused_shared_experts,
|
||||
top_k=self.top_k,
|
||||
tp_rank=self.ep_rank if self.use_ep else self.tp_rank,
|
||||
tp_size=self.ep_size if self.use_ep else self.tp_size,
|
||||
shared_experts_score=1.0,
|
||||
max_num_tokens=vllm_config.scheduler_config.max_num_batched_tokens
|
||||
* dp_size,
|
||||
is_EP=self.use_ep,
|
||||
)
|
||||
self.local_num_experts += self.num_fused_shared_experts
|
||||
|
||||
@overload
|
||||
def weight_loader(
|
||||
self,
|
||||
@@ -1866,6 +1946,7 @@ class FusedMoE(CustomOp):
|
||||
global_num_experts: int | None = None,
|
||||
zero_expert_num: int | None = None,
|
||||
zero_expert_type: str | None = None,
|
||||
num_fused_shared_experts: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Route the input hidden states to the top-k experts based on the
|
||||
@@ -1900,7 +1981,16 @@ class FusedMoE(CustomOp):
|
||||
if use_grouped_topk:
|
||||
assert topk_group is not None
|
||||
assert num_expert_group is not None
|
||||
topk_weights, topk_ids = grouped_topk(
|
||||
if is_rocm_aiter_moe_enabled():
|
||||
if not is_rocm_aiter_fusion_shared_expert_enabled():
|
||||
assert num_fused_shared_experts == 0
|
||||
grouped_topk_impl = partial(
|
||||
grouped_topk_aiter,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
)
|
||||
else:
|
||||
grouped_topk_impl = grouped_topk
|
||||
topk_weights, topk_ids = grouped_topk_impl(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=top_k,
|
||||
@@ -2119,7 +2209,9 @@ class FusedMoE(CustomOp):
|
||||
renormalize=self.renormalize,
|
||||
use_grouped_topk=self.use_grouped_topk,
|
||||
global_num_experts=self.global_num_experts,
|
||||
expert_map=self.expert_map,
|
||||
expert_map=self.expert_map
|
||||
if not is_rocm_aiter_moe_enabled()
|
||||
else self.expert_mask,
|
||||
topk_group=self.topk_group,
|
||||
num_expert_group=self.num_expert_group,
|
||||
custom_routing_function=self.custom_routing_function,
|
||||
@@ -2244,7 +2336,9 @@ class FusedMoE(CustomOp):
|
||||
renormalize=self.renormalize,
|
||||
use_grouped_topk=self.use_grouped_topk,
|
||||
global_num_experts=self.global_num_experts,
|
||||
expert_map=self.expert_map,
|
||||
expert_map=self.expert_map
|
||||
if not is_rocm_aiter_moe_enabled()
|
||||
else self.expert_mask,
|
||||
topk_group=self.topk_group,
|
||||
num_expert_group=self.num_expert_group,
|
||||
custom_routing_function=self.custom_routing_function,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from enum import IntEnum
|
||||
from functools import cache
|
||||
from functools import cache, lru_cache
|
||||
|
||||
import torch
|
||||
|
||||
@@ -46,6 +46,69 @@ def is_rocm_aiter_moe_enabled() -> bool:
|
||||
)
|
||||
|
||||
|
||||
@cache
|
||||
def is_rocm_aiter_fusion_shared_expert_enabled() -> bool:
|
||||
return (
|
||||
envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS and is_rocm_aiter_moe_enabled()
|
||||
)
|
||||
|
||||
|
||||
aiter_topK_meta_data = None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def init_aiter_topK_meta_data(
|
||||
n_routed_experts: int,
|
||||
n_shared_experts: int,
|
||||
top_k: int,
|
||||
tp_rank: int,
|
||||
tp_size: int,
|
||||
shared_experts_score: float = 1.0,
|
||||
max_num_tokens: int = 32768,
|
||||
is_EP: bool = False,
|
||||
):
|
||||
global aiter_topK_meta_data
|
||||
fake_expertid = n_routed_experts + n_shared_experts
|
||||
|
||||
# all layers reuse same buffer
|
||||
# This extra element when EP is enabled is used as a sentinel
|
||||
# to mask out shared expert processing for tokens not owned by
|
||||
# the current EP rank. This is necessary to avoid double-processing
|
||||
# of shared experts.
|
||||
total_topk_ids = torch.empty(
|
||||
(max_num_tokens, top_k + n_shared_experts + is_EP),
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
ns_topk_ids, s_topk_ids = total_topk_ids.split(
|
||||
[top_k, n_shared_experts + is_EP], dim=1
|
||||
)
|
||||
shared_expert_ids = [n_routed_experts + i for i in range(n_shared_experts + is_EP)]
|
||||
if is_EP:
|
||||
s_topk_ids_list = [
|
||||
[fake_expertid] * (n_shared_experts + is_EP)
|
||||
] * max_num_tokens
|
||||
for i in range(tp_rank, max_num_tokens, tp_size):
|
||||
s_topk_ids_list[i] = shared_expert_ids
|
||||
else:
|
||||
s_topk_ids_list = [
|
||||
list(range(n_routed_experts, fake_expertid))
|
||||
] * max_num_tokens
|
||||
s_topk_ids[:] = torch.tensor(s_topk_ids_list, dtype=torch.int32, device="cuda")
|
||||
|
||||
total_topk_weights = torch.empty(
|
||||
(max_num_tokens, top_k + n_shared_experts + is_EP),
|
||||
dtype=torch.float32,
|
||||
device="cuda",
|
||||
)
|
||||
ns_topk_weights, s_topk_weights = total_topk_weights.split(
|
||||
[top_k, n_shared_experts + is_EP], dim=1
|
||||
)
|
||||
s_topk_weights.fill_(shared_experts_score)
|
||||
assert aiter_topK_meta_data is None, "AITER topK meta data is already initialized"
|
||||
aiter_topK_meta_data = (total_topk_weights, total_topk_ids)
|
||||
|
||||
|
||||
def rocm_aiter_asm_moe_tkw1_impl(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
@@ -300,11 +363,33 @@ def rocm_aiter_grouped_topk(
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
num_fused_shared_experts: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
token = hidden_states.shape[0]
|
||||
device = hidden_states.device
|
||||
topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device)
|
||||
topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device)
|
||||
if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0:
|
||||
assert aiter_topK_meta_data is not None, (
|
||||
"AITER topK meta data is not initialized. "
|
||||
"Please ensure that init_aiter_topK_meta_data "
|
||||
"is called before this function."
|
||||
)
|
||||
total_topk_weights, total_topk_ids = aiter_topK_meta_data
|
||||
assert total_topk_weights.shape[0] >= token, (
|
||||
f"AITER topK meta data support {total_topk_weights.shape[0]} "
|
||||
f"tokens which is determined by max_num_batched_tokens, "
|
||||
f"but got {token} tokens now."
|
||||
)
|
||||
total_topk_weights = total_topk_weights[:token]
|
||||
total_topk_ids = total_topk_ids[:token]
|
||||
topk_weights, _ = total_topk_weights.split(
|
||||
[topk, total_topk_weights.shape[1] - topk], dim=1
|
||||
)
|
||||
topk_ids, _ = total_topk_ids.split(
|
||||
[topk, total_topk_ids.shape[1] - topk], dim=1
|
||||
)
|
||||
else:
|
||||
topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device)
|
||||
topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device)
|
||||
|
||||
if e_score_correction_bias is not None:
|
||||
torch.ops.vllm.rocm_aiter_biased_grouped_topk(
|
||||
@@ -315,6 +400,7 @@ def rocm_aiter_grouped_topk(
|
||||
num_expert_group,
|
||||
topk_group,
|
||||
renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
else:
|
||||
assert scoring_func == "softmax" or scoring_func == "sigmoid"
|
||||
@@ -326,10 +412,11 @@ def rocm_aiter_grouped_topk(
|
||||
topk_group,
|
||||
renormalize,
|
||||
scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights = topk_weights * routed_scaling_factor
|
||||
if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0:
|
||||
return total_topk_weights, total_topk_ids
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
@@ -354,7 +441,7 @@ def rocm_aiter_fused_experts(
|
||||
topk_weights = topk_weights.to(torch.float32)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
expert_mask = (expert_map > -1).to(torch.int32) if expert_map is not None else None
|
||||
expert_mask = expert_map if expert_map is not None else None
|
||||
|
||||
# w8a8 per-channel quantization
|
||||
if (
|
||||
|
||||
@@ -1056,6 +1056,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
indices_type=self.topk_indices_dtype,
|
||||
num_fused_shared_experts=layer.num_fused_shared_experts,
|
||||
)
|
||||
|
||||
per_act_token = self.input_quant.strategy == QuantizationStrategy.TOKEN
|
||||
|
||||
@@ -1169,6 +1169,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
global_num_experts=global_num_experts,
|
||||
zero_expert_num=zero_expert_num,
|
||||
zero_expert_type=zero_expert_type,
|
||||
num_fused_shared_experts=layer.num_fused_shared_experts,
|
||||
)
|
||||
|
||||
#
|
||||
|
||||
Reference in New Issue
Block a user