[Model] Kolibri 1 via the Transformers modeling backend (#60827)
Signed-off-by: Luca Motz <luca.motz@icloud.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Codex
Harry Mellor
parent
e90158d912
commit
698e79f3bd
@@ -144,6 +144,16 @@ class DisconnectedRouter(TopKRouter):
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class ExtraInputRouter(TopKRouter):
|
||||
"""Router taking a block-owned correction bias as a second input (HYV3)."""
|
||||
|
||||
def forward(self, hidden_states, bias=0.0):
|
||||
logits = F.linear(hidden_states, self.weight)
|
||||
_, index = torch.topk(logits.sigmoid() + bias, self.top_k, dim=-1)
|
||||
value = logits.sigmoid().gather(1, index)
|
||||
return logits, value, index
|
||||
|
||||
|
||||
class MoEExperts(nn.Module):
|
||||
"""Packed experts (3D weights); only its name (`experts`) matters here."""
|
||||
|
||||
@@ -166,17 +176,19 @@ class MoEBlock(nn.Module):
|
||||
|
||||
def _shared(self, x, logits):
|
||||
"""The term added to the experts' output (none for a plain block)."""
|
||||
return 0
|
||||
return None
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
logits, weights, index = self.gate(x)
|
||||
out = self.experts(x, index, weights) + self._shared(x, logits)
|
||||
out = self.experts(x, index, weights)
|
||||
if (shared := self._shared(x, logits)) is not None:
|
||||
out = out + shared
|
||||
return out.reshape(hidden_states.shape)
|
||||
|
||||
|
||||
class MoEBlockNoShared(MoEBlock):
|
||||
"""No shared-expert child but a gate-derived add -> trace skipped, still fuses."""
|
||||
"""No shared-expert child but a gate-derived add -> declined, not misread."""
|
||||
|
||||
def _shared(self, x, logits):
|
||||
return logits.sum()
|
||||
@@ -293,6 +305,115 @@ class MoEBlockUnaccountedBuffer(MoEBlockUnaccounted):
|
||||
self.extra = BufferScale()
|
||||
|
||||
|
||||
class MoEBlockExpertsKeywords(MoEBlock):
|
||||
"""Passes the experts' routing by keyword -> still fuses."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, weights=weights, index=index)
|
||||
|
||||
|
||||
class ScaledExperts(MoEExperts):
|
||||
def forward(self, hidden_states, index, weights, scale=1.0):
|
||||
return hidden_states * scale
|
||||
|
||||
|
||||
class MoEBlockExpertsExtraArg(MoEBlock):
|
||||
"""Passes the experts an extra argument the fused forward would drop."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.experts = ScaledExperts()
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, index, weights, scale=2.0)
|
||||
|
||||
|
||||
class MoEBlockSharedUnflattened(MoEBlockSharedNoGate):
|
||||
"""Gate and shared expert read the unflattened input (Kolibri-1)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
_, weights, index = self.gate(hidden_states)
|
||||
x = hidden_states.view(-1, hidden_states.shape[-1])
|
||||
out = self.experts(x, index, weights).view(*hidden_states.shape)
|
||||
return out + self.shared_expert(hidden_states)
|
||||
|
||||
|
||||
class MoEBlockSharedAveraged(MoEBlockSharedUnflattened):
|
||||
"""Averages the experts' and shared expert's outputs (Cohere2-MoE)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
return super().forward(hidden_states) / 2
|
||||
|
||||
|
||||
class MoEBlockScaled(MoEBlockSharedNoGate):
|
||||
"""Scales the experts' output by `routed_scaling_factor` (Laguna)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
return self.experts(x, index, weights) * 2.5 + self.shared_expert(x)
|
||||
|
||||
|
||||
class MoEBlockScaledBiasedRouter(MoEBlockScaled):
|
||||
"""An unrecognized router with a scaled experts' output."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.gate = BiasedRouter()
|
||||
|
||||
|
||||
class MoEBlockSharedBiasedRouter(MoEBlockSharedUnflattened):
|
||||
"""An unrecognized router with a shared expert."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.gate = BiasedRouter()
|
||||
|
||||
|
||||
class MoEBlockGateExtraInput(MoEBlock):
|
||||
"""Passes the gate a block-owned bias the fused forward would drop (HYV3)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(ExtraInputRouter)
|
||||
self.register_buffer("e_score_correction_bias", torch.zeros(8))
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x, self.e_score_correction_bias)
|
||||
return self.experts(x, index, weights)
|
||||
|
||||
|
||||
class MoEBlockCastCombine(MoEBlockSharedNoGate):
|
||||
"""Combines the outputs in fp32 (HYV3)."""
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
out = self.experts(x, index, weights).float() + self.shared_expert(x).float()
|
||||
return out.to(x.dtype)
|
||||
|
||||
|
||||
class MoEBlockIdentityLatent(MoEBlockSharedNoGate):
|
||||
"""Wraps the experts in unused (identity) latent projections (NemotronH)."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.fc1_latent_proj = nn.Identity()
|
||||
self.fc2_latent_proj = nn.Identity()
|
||||
|
||||
def forward(self, hidden_states):
|
||||
x = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
_, weights, index = self.gate(x)
|
||||
out = self.fc2_latent_proj(
|
||||
self.experts(self.fc1_latent_proj(x), index, weights)
|
||||
)
|
||||
return out + self.shared_expert(x)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sigmoid", [False, True])
|
||||
def test_moe_fuser_detects_router(sigmoid):
|
||||
with torch.device("meta"):
|
||||
@@ -385,16 +506,6 @@ def test_moe_fuser_detects_shared_experts():
|
||||
assert fuser.shared_gate_name == "shared_expert_gate"
|
||||
|
||||
|
||||
def test_moe_fuser_skips_shared_detection_without_extra_children():
|
||||
"""With only experts and gate, shared-expert detection (and its block trace)
|
||||
is skipped, so a gate-derived add is not misread as a shared expert."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockNoShared()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert isinstance(fuser, MoEBlockFuser)
|
||||
assert fuser.shared_name is None and fuser.shared_gate_name is None
|
||||
|
||||
|
||||
def test_moe_fuser_shared_without_gate():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockSharedNoGate()
|
||||
@@ -422,6 +533,10 @@ def test_moe_fuser_detects_non_glu_shared_expert():
|
||||
MoEBlockTupleVar, # tuple returned via a name binding, not a literal
|
||||
MoEBlockUnaccounted, # weight-bearing child outside the fused dataflow
|
||||
MoEBlockUnaccountedBuffer, # buffer-only child outside the fused dataflow
|
||||
MoEBlockExpertsExtraArg, # extra experts argument the fused forward drops
|
||||
MoEBlockNoShared, # gate-derived term added to the experts' output
|
||||
MoEBlockSharedAveraged, # experts' and shared expert's outputs averaged
|
||||
MoEBlockGateExtraInput, # gate input the fused forward drops
|
||||
],
|
||||
)
|
||||
def test_moe_fuser_declines_unsupported(block_cls):
|
||||
@@ -444,3 +559,53 @@ def test_moe_fuser_router_requires_connected_dataflow():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(DisconnectedRouter)
|
||||
assert MoEBlockFuser.match(block, "experts") is None
|
||||
|
||||
|
||||
def test_moe_fuser_unrecognized_router_moves_shared_expert():
|
||||
"""Without vLLM routing, the shared expert still moves to vLLM."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockSharedBiasedRouter()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
assert fuser.scoring_func is None
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("block_cls", "scoring_func"),
|
||||
[
|
||||
(MoEBlockSharedUnflattened, None), # routing falls back to Transformers
|
||||
(MoEBlock, "softmax"), # no shared output to divide
|
||||
],
|
||||
)
|
||||
def test_moe_fuser_divided_shared_output(block_cls, scoring_func):
|
||||
with torch.device("meta"):
|
||||
fuser = MoEBlockFuser.match(block_cls(), "experts", divides_shared_output=True)
|
||||
assert fuser.scoring_func == scoring_func
|
||||
|
||||
|
||||
def test_moe_fuser_scaled_output_needs_vllm_routing():
|
||||
"""Only vLLM routing applies the routed scale, so nothing else moves without it."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockScaledBiasedRouter()
|
||||
assert MoEBlockFuser.match(block, "experts") is None
|
||||
|
||||
|
||||
def test_moe_fuser_matches_experts_called_by_keyword():
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockExpertsKeywords()
|
||||
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
|
||||
|
||||
|
||||
def test_moe_fuser_scaled_output_with_vllm_routing():
|
||||
"""A scaled block fuses because vLLM routing applies `routed_scaling_factor`."""
|
||||
with torch.device("meta"):
|
||||
fuser = MoEBlockFuser.match(MoEBlockScaled(), "experts")
|
||||
assert fuser.scoring_func == "softmax"
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_cls", [MoEBlockCastCombine, MoEBlockIdentityLatent])
|
||||
def test_moe_fuser_looks_through_casts_and_identity(block_cls):
|
||||
with torch.device("meta"):
|
||||
fuser = MoEBlockFuser.match(block_cls(), "experts")
|
||||
assert fuser.shared_name == "shared_expert"
|
||||
|
||||
Reference in New Issue
Block a user