Fix Transformers backend FP8 MoE and remove some boilerplate (#46820)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -919,17 +919,17 @@ class Step3p5ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
|
||||
)
|
||||
|
||||
# Set MoE hyperparameters
|
||||
self.moe_layers: list[FusedMoEBlock] = []
|
||||
self.moe_layers: list[MoERunner] = []
|
||||
example_layer: FusedMoEBlock | None = None
|
||||
for layer in self.model.layers:
|
||||
if isinstance(layer, PPMissingLayer):
|
||||
continue
|
||||
assert isinstance(layer, Step3p5DecoderLayer)
|
||||
if hasattr(layer, "moe") and isinstance(layer.moe, FusedMoEBlock):
|
||||
self.moe_layers.append(layer.moe)
|
||||
example_layer = layer.moe
|
||||
self.moe_layers.append(layer.moe.experts)
|
||||
|
||||
self.expert_weights = []
|
||||
assert len(self.moe_layers) > 0, "No MoE layers found in the model."
|
||||
example_layer = self.moe_layers[0]
|
||||
self.num_moe_layers = len(self.moe_layers)
|
||||
self.num_expert_groups = 1
|
||||
self.num_shared_experts = 0
|
||||
@@ -959,24 +959,6 @@ class Step3p5ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_tokens(input_ids)
|
||||
|
||||
def set_eplb_state(
|
||||
self,
|
||||
expert_load_view: torch.Tensor,
|
||||
logical_to_physical_map: torch.Tensor,
|
||||
logical_replica_count: torch.Tensor,
|
||||
) -> None:
|
||||
for layer_idx, layer in enumerate(self.moe_layers):
|
||||
experts = layer.experts
|
||||
assert isinstance(experts, MoERunner)
|
||||
# Register the expert weights.
|
||||
self.expert_weights.append(experts.get_expert_weights())
|
||||
experts.set_eplb_state(
|
||||
moe_layer_idx=layer_idx,
|
||||
expert_load_view=expert_load_view,
|
||||
logical_to_physical_map=logical_to_physical_map,
|
||||
logical_replica_count=logical_replica_count,
|
||||
)
|
||||
|
||||
def update_physical_experts_metadata(
|
||||
self,
|
||||
num_physical_experts: int,
|
||||
|
||||
Reference in New Issue
Block a user