[V1] EP/TP MoE + DP Attention (#13931)
This commit is contained in:
@@ -46,7 +46,7 @@ class AriaImagePixelInputs(TypedDict):
|
||||
pixel_values: torch.Tensor
|
||||
pixel_mask: Optional[torch.Tensor]
|
||||
"""
|
||||
Shape:
|
||||
Shape:
|
||||
pixel_values: `(batch_size * num_images, num_channels, height, width)`
|
||||
pixel_mask: `(batch_size * num_images, height, width)`
|
||||
"""
|
||||
@@ -135,11 +135,11 @@ class AriaProjector(nn.Module):
|
||||
query numbers,
|
||||
e.g., {1225: 128, 4900: 256}. This allows for different query sizes
|
||||
based on image resolution.
|
||||
embed_dim (int): Embedding dimension.
|
||||
num_heads (int): Number of attention heads.
|
||||
kv_dim (int): Dimension of key and value.
|
||||
ff_dim (int): Hidden dimension of the feed-forward network.
|
||||
output_dim (int): Output dimension.
|
||||
embed_dim (int): Embedding dimension.
|
||||
num_heads (int): Number of attention heads.
|
||||
kv_dim (int): Dimension of key and value.
|
||||
ff_dim (int): Hidden dimension of the feed-forward network.
|
||||
output_dim (int): Output dimension.
|
||||
norm_layer (nn.Module): Normalization layer. Default is nn.LayerNorm.
|
||||
|
||||
Outputs:
|
||||
@@ -239,6 +239,7 @@ class AriaTextMoELayer(nn.Module):
|
||||
self,
|
||||
config: AriaTextConfig,
|
||||
quant_config: Optional[QuantizationConfig],
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -254,6 +255,7 @@ class AriaTextMoELayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
reduce_results=True,
|
||||
prefix=f"{prefix}.experts",
|
||||
)
|
||||
self.shared_experts = LlamaMLP(
|
||||
config.hidden_size,
|
||||
@@ -301,7 +303,9 @@ class AriaTextDecoderLayer(LlamaDecoderLayer):
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(config, cache_config, quant_config, prefix)
|
||||
self.mlp = AriaTextMoELayer(config, quant_config=quant_config)
|
||||
self.mlp = AriaTextMoELayer(config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp")
|
||||
|
||||
|
||||
class AriaTextModel(LlamaModel, SupportsQuant):
|
||||
|
||||
@@ -65,6 +65,7 @@ class DbrxExperts(FusedMoE):
|
||||
config: DbrxConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
params_dtype: Optional[torch.dtype] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__(
|
||||
num_experts=config.ffn_config.moe_num_experts,
|
||||
@@ -76,6 +77,7 @@ class DbrxExperts(FusedMoE):
|
||||
renormalize=True,
|
||||
quant_config=quant_config,
|
||||
tp_size=get_tensor_model_parallel_world_size(),
|
||||
prefix=prefix,
|
||||
)
|
||||
self.config = config
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
@@ -139,6 +141,7 @@ class DbrxMoE(nn.Module):
|
||||
config: DbrxConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
params_dtype: Optional[torch.dtype] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.d_model = config.d_model
|
||||
@@ -150,7 +153,8 @@ class DbrxMoE(nn.Module):
|
||||
|
||||
self.experts = DbrxExperts(config=config,
|
||||
quant_config=quant_config,
|
||||
params_dtype=self.params_dtype)
|
||||
params_dtype=self.params_dtype,
|
||||
prefix=f"{prefix}.experts")
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
orig_shape = hidden_states.shape
|
||||
@@ -291,7 +295,7 @@ class DbrxBlock(nn.Module):
|
||||
cache_config,
|
||||
quant_config,
|
||||
prefix=f"{prefix}.norm_attn_norm")
|
||||
self.ffn = DbrxMoE(config, quant_config)
|
||||
self.ffn = DbrxMoE(config, quant_config, prefix=f"{prefix}.ffn")
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -47,7 +47,8 @@ class JambaMoE(nn.Module):
|
||||
top_k: Optional[int] = None,
|
||||
params_dtype: Optional[torch.dtype] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = ""):
|
||||
super().__init__()
|
||||
self.num_total_experts = num_experts or config.num_experts
|
||||
self.top_k = top_k or config.num_experts_per_tok
|
||||
@@ -70,7 +71,8 @@ class JambaMoE(nn.Module):
|
||||
reduce_results=True,
|
||||
renormalize=False,
|
||||
use_grouped_topk=False,
|
||||
quant_config=quant_config)
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts")
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
orig_shape = hidden_states.shape
|
||||
@@ -92,13 +94,15 @@ class JambaMLP(JambaMoE):
|
||||
config: JambaConfig,
|
||||
params_dtype: Optional[torch.dtype] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = ""):
|
||||
super().__init__(config,
|
||||
num_experts=1,
|
||||
top_k=1,
|
||||
params_dtype=params_dtype,
|
||||
tp_size=tp_size,
|
||||
quant_config=quant_config)
|
||||
quant_config=quant_config,
|
||||
prefix=prefix)
|
||||
|
||||
|
||||
class JambaMambaDecoderLayer(nn.Module):
|
||||
@@ -109,6 +113,7 @@ class JambaMambaDecoderLayer(nn.Module):
|
||||
cache_config: Optional[CacheConfig] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
is_lora_enabled: Optional[bool] = False,
|
||||
prefix: str = "",
|
||||
**kwargs) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -129,7 +134,9 @@ class JambaMambaDecoderLayer(nn.Module):
|
||||
|
||||
num_experts = config.layers_num_experts[layer_idx]
|
||||
ffn_layer_class = JambaMoE if num_experts > 1 else JambaMLP
|
||||
self.feed_forward = ffn_layer_class(config, quant_config=quant_config)
|
||||
self.feed_forward = ffn_layer_class(config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.feed_forward")
|
||||
self.input_layernorm = RMSNorm(config.hidden_size,
|
||||
eps=config.rms_norm_eps)
|
||||
self.pre_ff_layernorm = RMSNorm(config.hidden_size,
|
||||
@@ -211,7 +218,9 @@ class JambaAttentionDecoderLayer(nn.Module):
|
||||
|
||||
num_experts = config.layers_num_experts[layer_idx]
|
||||
ffn_layer_class = JambaMoE if num_experts > 1 else JambaMLP
|
||||
self.feed_forward = ffn_layer_class(config, quant_config=quant_config)
|
||||
self.feed_forward = ffn_layer_class(config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.feed_forward")
|
||||
self.input_layernorm = RMSNorm(config.hidden_size,
|
||||
eps=config.rms_norm_eps)
|
||||
self.pre_ff_layernorm = RMSNorm(config.hidden_size,
|
||||
|
||||
@@ -71,6 +71,7 @@ class MixtralMoE(nn.Module):
|
||||
params_dtype: Optional[torch.dtype] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
dp_size: Optional[int] = None,
|
||||
prefix: str = ""):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -93,6 +94,7 @@ class MixtralMoE(nn.Module):
|
||||
renormalize=True,
|
||||
quant_config=quant_config,
|
||||
tp_size=tp_size,
|
||||
dp_size=dp_size,
|
||||
prefix=f"{prefix}.experts")
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -80,7 +80,8 @@ class OlmoeMoE(nn.Module):
|
||||
reduce_results=True,
|
||||
renormalize=False,
|
||||
quant_config=quant_config,
|
||||
tp_size=tp_size)
|
||||
tp_size=tp_size,
|
||||
prefix=f"{prefix}.experts")
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
# NOTE: hidden_states can have either 1D or 2D shape.
|
||||
@@ -212,6 +213,7 @@ class OlmoeDecoderLayer(nn.Module):
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=1e-5)
|
||||
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=1e-5)
|
||||
|
||||
@@ -249,6 +249,7 @@ class PhiMoE(nn.Module):
|
||||
params_dtype: Optional[torch.dtype] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -272,7 +273,8 @@ class PhiMoE(nn.Module):
|
||||
renormalize=False,
|
||||
quant_config=quant_config,
|
||||
tp_size=tp_size,
|
||||
custom_routing_function=phimoe_routing_function)
|
||||
custom_routing_function=phimoe_routing_function,
|
||||
prefix=f"{prefix}.experts")
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
# NOTE: hidden_states can have either 1D or 2D shape.
|
||||
@@ -396,6 +398,7 @@ class PhiMoEDecoderLayer(nn.Module):
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.block_sparse_moe",
|
||||
)
|
||||
self.input_layernorm = nn.LayerNorm(config.hidden_size,
|
||||
eps=config.rms_norm_eps,
|
||||
|
||||
@@ -100,6 +100,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
@@ -115,7 +116,8 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
reduce_results=False,
|
||||
renormalize=config.norm_topk_prob,
|
||||
quant_config=quant_config)
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts")
|
||||
|
||||
self.gate = ReplicatedLinear(config.hidden_size,
|
||||
config.num_experts,
|
||||
@@ -277,7 +279,8 @@ class Qwen2MoeDecoderLayer(nn.Module):
|
||||
config.num_experts > 0 and
|
||||
(layer_idx + 1) % config.decoder_sparse_step == 0):
|
||||
self.mlp = Qwen2MoeSparseMoeBlock(config=config,
|
||||
quant_config=quant_config)
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp")
|
||||
else:
|
||||
self.mlp = Qwen2MoeMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
|
||||
Reference in New Issue
Block a user