Signed-off-by: Xin Yang <xyangx@amazon.com>
This commit is contained in:
@@ -20,11 +20,12 @@ from vllm.distributed import (
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, GateLinear
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
@@ -174,11 +175,13 @@ class MLPBlock(torch.nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
self.experts_per_token = config.num_experts_per_tok
|
||||
self.world_size = dist.get_world_size() if dist.is_initialized() else 1
|
||||
self.router = GateLinear(
|
||||
self.router = ReplicatedLinear(
|
||||
config.hidden_size,
|
||||
config.num_local_experts,
|
||||
bias=True,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.router",
|
||||
return_bias=False,
|
||||
)
|
||||
assert config.intermediate_size % self.world_size == 0
|
||||
self.experts = FusedMoE(
|
||||
@@ -206,7 +209,7 @@ class MLPBlock(torch.nn.Module):
|
||||
self, x[:, : self.hidden_size], self.router.weight, self.router.bias
|
||||
)
|
||||
else:
|
||||
g, _ = self.router(x)
|
||||
g = self.router(x)
|
||||
x = self.experts(hidden_states=x, router_logits=g)[:, : self.hidden_size]
|
||||
|
||||
if self.is_sequence_parallel:
|
||||
|
||||
Reference in New Issue
Block a user