[Model] Add Inkling LoRA support [4/N] (#48884)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Woosuk Kwon
2026-07-17 09:52:15 +08:00
committed by GitHub
co-authored by Bugen Zhao Giancarlo Delfin Isotr0py Isotr0py Jee Jee Li Roger Wang Yifan Qiao Claude Fable 5 OpenAI Codex
parent fe784ff22e
commit f3e9497e92
12 changed files with 337 additions and 47 deletions
@@ -6,7 +6,9 @@ from types import SimpleNamespace
import pytest
import torch
from vllm.lora.utils import get_supported_lora_modules
from vllm.models.inkling.nvidia import moe
from vllm.models.inkling.nvidia.model import _TmlForCausalLMBase
from vllm.platforms import current_platform
@@ -62,6 +64,29 @@ def test_gate_uses_ll_bf16_gemm_through_token_limit(
assert logits.dtype == torch.float32
def test_gate_is_not_a_lora_target() -> None:
model = torch.nn.Module()
model.gate = moe.InklingGate(
d_model=4,
n_routed_experts=5,
n_shared_experts=2,
experts_per_token=2,
route_scale=1.0,
)
assert "gate" not in get_supported_lora_modules(model)
def test_custom_embedding_is_not_a_lora_target() -> None:
model = torch.nn.Module()
model.embedding_modules = _TmlForCausalLMBase.embedding_modules
supported = get_supported_lora_modules(model)
assert "embed_tokens" not in supported
assert "lm_head" in supported
@pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)])
def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
experts = SimpleNamespace(
+19 -1
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from typing import TYPE_CHECKING, Any, Literal
import torch
@@ -75,8 +76,14 @@ class LoRAConfig:
"""If True, force the engine to use the universal 2D MoE LoRA wrapper
(`FusedMoEWithLoRA`) regardless of the model's `is_3d_moe_weight` flag, so
that 2D-format and 3D-format MoE LoRA adapters can be served in the same
deployment. Only meaningful forMoE models; ignored otherwise. Default False
deployment. Only meaningful for MoE models; ignored otherwise. Default False
keeps the existing model-driven behavior."""
enable_moe_shared_loras: bool = False
"""If True, load MoE expert adapters in the "shared-outer" layout, where the
gate/up (`w1`/`w3`) lora_A and the down (`w2`) lora_B are shared across all
experts (stored once with expert-dim 1) instead of per-expert. The shared
factors are broadcast to the expert count at kernel time. Only meaningful for
MoE models whose adapters use this layout; ignored otherwise."""
def compute_hash(self) -> str:
"""
@@ -97,6 +104,7 @@ class LoRAConfig:
factors.append(self.lora_dtype)
factors.append(self.enable_tower_connector_lora)
factors.append(self.enable_mixed_moe_lora_format)
factors.append(self.enable_moe_shared_loras)
# target_modules affects which modules get LoRA applied
factors.append(
tuple(sorted(self.target_modules)) if self.target_modules else None
@@ -129,3 +137,13 @@ class LoRAConfig:
self.lora_dtype = model_config.dtype
elif isinstance(self.lora_dtype, str):
self.lora_dtype = getattr(torch, self.lora_dtype)
architectures = getattr(model_config, "architectures", None) or []
is_inkling = any("Inkling" in arch for arch in architectures)
if is_inkling and os.environ.get("INKLING_MULTIMEM_AR", "1") != "0":
raise ValueError(
"Inkling LoRA requires INKLING_MULTIMEM_AR=0: the Lamport "
"fused-collective path bypasses the LoRA-wrapped wo_ud and dense "
"down_proj layers on decode-sized batches, silently dropping "
"their LoRA. Set INKLING_MULTIMEM_AR=0."
)
+6
View File
@@ -602,6 +602,7 @@ class EngineArgs:
enable_tower_connector_lora: bool = LoRAConfig.enable_tower_connector_lora
specialize_active_lora: bool = LoRAConfig.specialize_active_lora
enable_mixed_moe_lora_format: bool = LoRAConfig.enable_mixed_moe_lora_format
enable_moe_shared_loras: bool = LoRAConfig.enable_moe_shared_loras
ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight
num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override
@@ -1351,6 +1352,10 @@ class EngineArgs:
"--enable-mixed-moe-lora-format",
**lora_kwargs["enable_mixed_moe_lora_format"],
)
lora_group.add_argument(
"--enable-moe-shared-loras",
**lora_kwargs["enable_moe_shared_loras"],
)
# Observability arguments
observability_kwargs = get_kwargs(ObservabilityConfig)
@@ -2213,6 +2218,7 @@ class EngineArgs:
enable_tower_connector_lora=self.enable_tower_connector_lora,
specialize_active_lora=self.specialize_active_lora,
enable_mixed_moe_lora_format=self.enable_mixed_moe_lora_format,
enable_moe_shared_loras=self.enable_moe_shared_loras,
max_cpu_loras=self.max_cpu_loras
if self.max_cpu_loras and self.max_cpu_loras > 0
else None,
+51 -15
View File
@@ -57,6 +57,9 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
# For non-gated MoE (is_act_and_mul=False), only 1 slice is needed
# since there's only up_proj (w1), not gate_proj + up_proj (w1 + w3)
self._w13_slices = 2 if base_layer.moe_config.is_act_and_mul else 1
# Set from lora_config.enable_moe_shared_loras in create_lora_weights.
# When True, w13 lora_A and w2 lora_B are stored once for all experts.
self.enable_moe_shared_loras = False
# Mirrors per-(lora_id) layout of `self.lora_a_stacked` (built in
# `create_lora_weights`) so `create_dummy_lora`'s n_slices fallback
# matches `lora_a_stacked` length under EP.
@@ -150,10 +153,21 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
local_num_experts=self.local_num_experts,
punica_wrapper=self.punica_wrapper,
use_tuned_config=bool(envs.VLLM_TUNED_CONFIG_FOLDER),
enable_moe_shared_loras=self.enable_moe_shared_loras,
aux_stream=self._lora_stream if use_dual_stream else None,
events=self._events if use_dual_stream else None,
)
@property
def _w13_a_num_experts(self) -> int:
"""Expert-dim of the w13 lora_A buffer: 1 when shared."""
return 1 if self.enable_moe_shared_loras else self.local_num_experts
@property
def _w2_b_num_experts(self) -> int:
"""Expert-dim of the w2 lora_B buffer: 1 when shared."""
return 1 if self.enable_moe_shared_loras else self.local_num_experts
def _create_lora_a_weights(
self,
max_loras: int,
@@ -163,7 +177,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
torch.zeros(
(
max_loras,
self.local_num_experts,
self._w13_a_num_experts,
lora_config.max_lora_rank
if not self.fully_sharded
else divide(lora_config.max_lora_rank, self.tp_size),
@@ -205,7 +219,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
torch.zeros(
(
max_loras,
self.local_num_experts,
self._w2_b_num_experts,
self.hidden_size
if not self.fully_sharded
else divide(self.hidden_size, self.tp_size),
@@ -247,6 +261,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
self._verify_ep_fs(lora_config)
self.max_loras = lora_config.max_loras
self.fully_sharded = lora_config.fully_sharded_loras
self.enable_moe_shared_loras = lora_config.enable_moe_shared_loras
self.adapter_enabled = torch.tensor(
[0] * (max_loras + 1), dtype=torch.int, device=self.device
@@ -263,8 +278,11 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
for experts_id in range(self.local_num_experts):
# For gated MoE: gate_proj (w1), down_proj (w2), up_proj (w3)
# For non-gated MoE: up_proj (w1), down_proj (w2)
# Shared factors are collapsed (expert-dim 1): index 0.
w13_a_eid = 0 if self.enable_moe_shared_loras else experts_id
w2_b_eid = 0 if self.enable_moe_shared_loras else experts_id
self.lora_a_stacked.append(
self.w13_lora_a_stacked[0][lora_id][experts_id]
self.w13_lora_a_stacked[0][lora_id][w13_a_eid]
)
self.lora_a_stacked.append(
self.w2_lora_a_stacked[0][lora_id][experts_id]
@@ -273,14 +291,12 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
self.lora_b_stacked.append(
self.w13_lora_b_stacked[0][lora_id][experts_id]
)
self.lora_b_stacked.append(
self.w2_lora_b_stacked[0][lora_id][experts_id]
)
self.lora_b_stacked.append(self.w2_lora_b_stacked[0][lora_id][w2_b_eid])
# Only add w3 (up_proj) for gated MoE (_w13_slices == 2)
if self._w13_slices == 2:
self.lora_a_stacked.append(
self.w13_lora_a_stacked[1][lora_id][experts_id]
self.w13_lora_a_stacked[1][lora_id][w13_a_eid]
)
self.lora_b_stacked.append(
self.w13_lora_b_stacked[1][lora_id][experts_id]
@@ -340,6 +356,22 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
return w2_lora_b[:, start_idx:end_idx, :]
def _match_expert_dim(
self, src: torch.Tensor, buffer: torch.Tensor
) -> torch.Tensor:
"""Align a LoRA factor's expert-dim (dim 0) to its stacked buffer.
Equal dims pass through. When the buffer is collapsed to expert-dim 1
(a shared factor) but the source carries per-expert copies, keep
the first — every expert shares the same factor, so the copies are
identical for a real adapter and irrelevant (zeros) for the dummy.
"""
tgt = buffer.shape[1]
if src.shape[0] == tgt:
return src
assert tgt == 1, f"expert-dim mismatch: source {src.shape[0]} vs buffer {tgt}"
return src[:1]
def reset_lora(self, index: int):
"""Resets the lora weights at index back to 0."""
for pos in range(self._w13_slices):
@@ -366,20 +398,24 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
self.reset_lora(index)
self.adapter_enabled[index] = 1
num_experts = self.w13_lora_a_stacked[0].shape[1]
w1_lora_a, w2_lora_a, w3_lora_a = lora_a
w1_lora_b, w2_lora_b, w3_lora_b = lora_b
# EP slicing is done once at add time in
# LoRAModelManager._slice_moe_lora_ep, so by here the cached
# tensors already match the local-expert dim of the stacked buffers.
assert (
num_experts
== w1_lora_a.shape[0]
== w2_lora_a.shape[0]
== w3_lora_a.shape[0]
)
# For shared-loras adapters the w13 lora_A and w2 lora_B buffers are
# collapsed to expert-dim 1: match each factor to the expert-dim of
# the buffer it is copied into. A real shared factor already has
# expert-dim 1 (no-op); a per-expert source (e.g. the warmup dummy
# LoRA, which always stacks to num_experts) is collapsed to the shared
# slot since every expert shares the same factor.
w1_lora_a = self._match_expert_dim(w1_lora_a, self.w13_lora_a_stacked[0])
w3_lora_a = self._match_expert_dim(w3_lora_a, self.w13_lora_a_stacked[-1])
w2_lora_a = self._match_expert_dim(w2_lora_a, self.w2_lora_a_stacked[0])
w1_lora_b = self._match_expert_dim(w1_lora_b, self.w13_lora_b_stacked[0])
w3_lora_b = self._match_expert_dim(w3_lora_b, self.w13_lora_b_stacked[-1])
w2_lora_b = self._match_expert_dim(w2_lora_b, self.w2_lora_b_stacked[0])
slliced_w1_lora_a = self._slice_w13_a(w1_lora_a)
slliced_w1_lora_b = self._slice_w13_b(w1_lora_b)
+33
View File
@@ -227,6 +227,39 @@ class PackedLoRALayerWeights(LoRALayerWeights):
)
return obj
@classmethod
def pack_moe_stacked(
cls,
loras: GenericSequence["LoRALayerWeights | None"],
module_name: str,
) -> "PackedLoRALayerWeights":
"""Pack pre-stacked (3D) w1/w2/w3 expert LoRAs into a single LoRA.
Unlike :meth:`pack_moe`, which stacks one 2D tensor per expert, this
expects each of the three input LoRAs to already carry the expert
dimension (``experts.w{1,2,3}``). It is used for "shared-outer"
adapters where some factors are shared across experts (expert-dim 1)
and cannot be reconstructed by stacking per-expert tensors. The
produced ``lora_a``/``lora_b`` are the ``[w1, w2, w3]`` lists that
``FusedMoEWithLoRA.set_lora`` consumes directly.
"""
assert len(loras) == 3, (
"shared-outer expert LoRA expects exactly w1/w2/w3 stacked tensors"
)
w1_lora, w2_lora, w3_lora = loras
assert w1_lora is not None and w2_lora is not None and w3_lora is not None
rank = w1_lora.rank
lora_alpha = w1_lora.lora_alpha
scaling = lora_alpha / rank
return cls(
module_name,
rank,
[lora_alpha, lora_alpha, lora_alpha],
[w1_lora.lora_a, w2_lora.lora_a, w3_lora.lora_a],
[w1_lora.lora_b, w2_lora.lora_b, w3_lora.lora_b],
scaling=[scaling, scaling, scaling],
)
def optimize(self) -> "PackedLoRALayerWeights":
"""Optimize the LoRA by merging the scaling into lora_b."""
for i in range(len(self.lora_b)):
+26 -24
View File
@@ -132,8 +132,13 @@ class LoRAModelManager:
and self.model.is_3d_moe_weight
and not self._enable_mixed_moe_lora_format
)
# Shared MoE adapters: w13 lora_A / w2 lora_B shared across experts,
# stored as pre-stacked experts.w{1,2,3} tensors (startup opt-in).
self._enable_moe_shared_loras = is_moe and lora_config.enable_moe_shared_loras
self.packed_modules_mapping = process_packed_modules_mapping(
self.model, force_2d_moe=self._enable_mixed_moe_lora_format
self.model,
force_2d_moe=self._enable_mixed_moe_lora_format,
enable_moe_shared_loras=self._enable_moe_shared_loras,
)
self._is_non_gated_moe = is_moe and self.model.is_non_gated_moe
self._use_ep = bool(
@@ -761,6 +766,16 @@ class LoRAModelManager:
if lora_model.check_lora_name(replaced_module_name):
module_name = replaced_module_name
if module_name.endswith(".experts"):
if self._enable_moe_shared_loras:
lora_model.loras[module_name] = (
PackedLoRALayerWeights.pack_moe_stacked(
replacement_loras,
module_name,
)
)
for module in packed_module_names:
lora_model.loras.pop(module, None)
continue
if self._is_non_gated_moe and len(replacement_loras) > 0:
replacement_loras = self._pad_lora_pairs_to_triplets(
replacement_loras
@@ -1019,20 +1034,10 @@ class LoRAModelManager:
module: FusedMoEWithLoRA,
module_name: str,
) -> None:
"""Slice the cached LoRA tensors down to this rank's local experts.
"""Slice cached LoRA tensors down to this rank's local experts.
The 2D MoE checkpoint enters as a list of per-(w1/w2/w3) tensors of
shape (num_experts, rank, in) / (num_experts, out, rank). When EP
is active each rank only owns local_num_experts; without this slice
the CPU LoRAModel keeps the full global weight and set_lora has to
re-slice on every activation.
With the load-time / pack-time slicing in
``_restrict_to_local_experts``, the stacked tensors already match
``local_num_experts`` and the inner branch becomes a no-op. The
guard remains so checkpoints that bypassed the pre-slicing (e.g.
``.bin``/``.pt`` adapters with weights mappers we don't recognize)
still get sliced here.
Shared factors have expert dimension one and remain available on every
rank. Per-expert factors are narrowed to the local expert block.
"""
if not module.use_ep:
return
@@ -1046,16 +1051,13 @@ class LoRAModelManager:
expert_start = ep_rank * local_num_experts
expert_end = expert_start + local_num_experts
new_lora_a: list[torch.Tensor | None] = []
new_lora_b: list[torch.Tensor | None] = []
for a, b in zip(module_lora.lora_a, module_lora.lora_b):
if a is not None and b is not None and a.shape[0] == global_num_experts:
a = a[expert_start:expert_end].contiguous()
b = b[expert_start:expert_end].contiguous()
new_lora_a.append(a)
new_lora_b.append(b)
module_lora.lora_a = new_lora_a
module_lora.lora_b = new_lora_b
def _slice_local(t: torch.Tensor | None) -> torch.Tensor | None:
if t is not None and t.shape[0] == global_num_experts:
return t[expert_start:expert_end].contiguous()
return t
module_lora.lora_a = [_slice_local(a) for a in module_lora.lora_a]
module_lora.lora_b = [_slice_local(b) for b in module_lora.lora_b]
def _restrict_to_local_experts(
self, module_name: str, new_module_names: list[str]
+14 -2
View File
@@ -369,7 +369,9 @@ def get_adapter_absolute_path(lora_path: str) -> str:
def process_packed_modules_mapping(
model: nn.Module, force_2d_moe: bool = False
model: nn.Module,
force_2d_moe: bool = False,
enable_moe_shared_loras: bool = False,
) -> dict[str, list[str]]:
if is_moe_model(model):
# This method generates and returns a dictionary mapping packed module
@@ -382,7 +384,17 @@ def process_packed_modules_mapping(
# the engine forces the universal 2D wrapper via
# enable_mixed_moe_lora_format (so 3D models can also load 2D
# adapters through FusedMoEWithLoRA).
if (not model.is_3d_moe_weight) or force_2d_moe:
if enable_moe_shared_loras:
# Shared MoE adapters store one pre-stacked tensor per
# expert-projection (experts.w1/w2/w3) rather than a per-expert
# tensor list, so the packed mapping references the three stack
# names directly (drives expected_lora_modules and the packing).
packed_modules_mapping["experts"] = [
"experts.w1",
"experts.w2",
"experts.w3",
]
elif (not model.is_3d_moe_weight) or force_2d_moe:
# Filter out malformed entries: non-gated MoE has empty
# ckpt_up_proj_name which results in weight_name containing ".."
# (e.g., "experts.0.." instead of "experts.0.layer_name.")
@@ -43,6 +43,11 @@ class MoELoRAContext:
# try_get_optimal_moe_lora_config for Triton kernel tile configs.
use_tuned_config: bool
# Shared MoE LoRA: w13 lora_A and w2 lora_B are shared across all experts
# (stored collapsed with expert-dim 1). When True, LoRAExpertsMixin expands
# them to local_num_experts via a stride-0 view before the kernel.
enable_moe_shared_loras: bool = False
# Optional dual-stream support for overlapping each (base GEMM, LoRA)
# pair. When aux_stream is None, the experts.apply() path runs the
# original sequential schedule. When set, base GEMM runs on the default
@@ -52,10 +52,19 @@ class LoRAExpertsMixin:
torch.Tensor | None,
torch.Tensor | None,
]:
w13_lora_a_stacked = lora_context.w13_lora_a_stacked
if lora_context.enable_moe_shared_loras:
# w13 lora_A is shared across experts (collapsed expert-dim 1);
# broadcast to local_num_experts via a stride-0 view. The kernel
# derives num_experts from shape[1] and indexes via stride(1)==0.
w13_lora_a_stacked = tuple(
a.expand(-1, lora_context.local_num_experts, -1, -1)
for a in w13_lora_a_stacked
)
return lora_context.punica_wrapper.add_lora_w13(
y,
x,
lora_context.w13_lora_a_stacked,
w13_lora_a_stacked,
lora_context.w13_lora_b_stacked,
topk_ids,
topk_weights,
@@ -92,11 +101,19 @@ class LoRAExpertsMixin:
top_k_num: int,
add_inputs: bool = True,
) -> None:
w2_lora_b_stacked = lora_context.w2_lora_b_stacked
if lora_context.enable_moe_shared_loras:
# w2 lora_B is shared across experts (collapsed expert-dim 1);
# broadcast to local_num_experts via a stride-0 view.
w2_lora_b_stacked = tuple(
b.expand(-1, lora_context.local_num_experts, -1, -1)
for b in w2_lora_b_stacked
)
lora_context.punica_wrapper.add_lora_w2(
y,
x,
lora_context.w2_lora_a_stacked,
lora_context.w2_lora_b_stacked,
w2_lora_b_stacked,
topk_weights,
sorted_token_ids_lora,
expert_ids_lora,
+57 -1
View File
@@ -1,6 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inkling logits processor with muP output scaling."""
"""Inkling logits processor (muP + LoRA aware).
Inkling divides the final logits by a muP width multiplier
(``logits_mup_width_multiplier``). This applies it two ways, depending on
whether an lm_head LoRA is attached:
* No LoRA: fold ``1/mup`` into the lm_head GEMM alpha (fp32 epilogue) -- no
separate elementwise kernel, no extra rounding, no weight mutation.
* LoRA attached: the LoRA manager wraps this layer in
``LogitsProcessorWithLoRA``, whose ``forward`` calls
``type(base_layer).forward(self=wrapper)`` -- so this ``forward`` runs with
``self`` bound to the wrapper. We detect that via ``base_layer`` and take the
LoRA path: run the wrapper's ``_get_logits`` (base logits + the lm_head LoRA
delta), then divide the full logits by the multiplier so the delta is scaled
too. muP thus composes with the LoRA delta, with the dispatch as the only
model-side branching.
"""
from __future__ import annotations
@@ -47,6 +63,46 @@ class InklingLogitsProcessor(LogitsProcessor):
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None = None,
) -> torch.Tensor | None:
# ``base_layer`` exists only on the LogitsProcessorWithLoRA wrapper,
# which calls this forward with ``self`` bound to the wrapper. The
# wrapper is not an ``InklingLogitsProcessor`` instance, so dispatch
# ``_lora_forward`` explicitly through the base_layer's class (it
# provides ``_get_logits``/``logits_as_input``; only ``_lora_forward``
# lives on this class).
if hasattr(self, "base_layer"):
return type(self.base_layer)._lora_forward(
self, lm_head, hidden_states, embedding_bias
)
return self._base_forward(lm_head, hidden_states, embedding_bias)
def _lora_forward(
self,
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None = None,
) -> torch.Tensor | None:
# ``self`` is the LogitsProcessorWithLoRA wrapper here: ``_get_logits``
# returns the base logits plus the lm_head LoRA delta. Apply the muP
# divisor on the full logits so the LoRA delta is scaled too.
mup_multiplier = self.base_layer.logits_mup_width_multiplier
mup = 1.0 / mup_multiplier if mup_multiplier else None
if self.logits_as_input:
logits = hidden_states
else:
logits = self._get_logits(hidden_states, lm_head, embedding_bias)
# TODO: fuse this multiplication
if logits is not None and mup:
assert self.base_layer.soft_cap is None
assert self.base_layer.scale == 1.0
logits = logits * mup
return logits
def _base_forward(
self,
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None = None,
) -> torch.Tensor | None:
mup = self.logits_mup_width_multiplier
if not mup:
+11 -1
View File
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsLoRA,
SupportsMultiModal,
SupportsPP,
)
@@ -360,7 +361,7 @@ class InklingModel(nn.Module):
return self.norm(hidden_states)
class _TmlForCausalLMBase(nn.Module, SupportsPP):
class _TmlForCausalLMBase(nn.Module, SupportsPP, SupportsLoRA):
"""Shared text-backbone causal-LM scaffolding for both entry classes."""
hf_to_vllm_mapper = WeightsMapper(
@@ -380,6 +381,8 @@ class _TmlForCausalLMBase(nn.Module, SupportsPP):
"model.llm.embed": "model.embed_tokens",
"model.llm.norm": "model.norm",
"model.llm.unembed": "lm_head",
"language_model.layers.": "model.layers.",
"language_model.lm_head.": "lm_head.",
},
orig_to_new_suffix={
# NVFP4 scale
@@ -389,6 +392,13 @@ class _TmlForCausalLMBase(nn.Module, SupportsPP):
".w2_weight.scale2": ".w2_weight_scale_2",
},
)
packed_modules_mapping = {
"qkvr": ["wq_du", "wk_dv", "wv_dv", "wr_du"],
"w13": ["w1", "w3"],
}
embedding_modules = {
"lm_head": "output_embeddings",
}
def _build(
self,
+71 -1
View File
@@ -343,6 +343,71 @@ class InklingSinkExperts(nn.Module):
return h @ self.w2_weight.T # (T, D)
class InklingSinkExpertsLinear(nn.Module):
"""LoRA-capable implementation of the Inkling sink experts."""
def __init__(
self,
n_experts: int,
d_model: int,
d_mlp: int,
*,
prefix: str = "",
) -> None:
super().__init__()
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
RowParallelLinear,
)
self.n_experts = n_experts
self.d_mlp = d_mlp
total = n_experts * d_mlp
self.w13 = MergedColumnParallelLinear(
input_size=d_model,
output_sizes=[total, total],
bias=False,
prefix=f"{prefix}.w13",
)
self.w2 = RowParallelLinear(
input_size=total,
output_size=d_model,
bias=False,
reduce_results=False,
prefix=f"{prefix}.w2",
)
self._w2_input_pp = self.w2.input_size_per_partition
self._col_expert: torch.Tensor | None = None
def _gamma_expand(self, gammas: torch.Tensor) -> torch.Tensor:
if self._col_expert is None or self._col_expert.device != gammas.device:
local = self._w2_input_pp
start = get_tensor_model_parallel_rank() * local
cols = torch.arange(start, start + local, device=gammas.device)
self._col_expert = (cols // self.d_mlp).long()
return gammas[:, self._col_expert]
def load_weight(self, key: str, weight: torch.Tensor) -> list[str]:
if key == "w13_weight":
d_model = weight.shape[-1]
gate = weight[:, 0::2, :].reshape(-1, d_model).contiguous()
up = weight[:, 1::2, :].reshape(-1, d_model).contiguous()
self.w13.weight_loader(self.w13.weight, gate, 0)
self.w13.weight_loader(self.w13.weight, up, 1)
return ["w13.weight"]
w = weight.permute(1, 0, 2).reshape(weight.shape[1], -1).contiguous()
self.w2.weight_loader(self.w2.weight, w)
return ["w2.weight"]
def forward(self, x: torch.Tensor, gammas: torch.Tensor) -> torch.Tensor:
gate_up, _ = self.w13(x)
gate, up = gate_up.chunk(2, dim=-1)
hidden_states = torch.nn.functional.silu(gate) * up
hidden_states = (hidden_states * self._gamma_expand(gammas)).to(x.dtype)
output, _ = self.w2(hidden_states)
return output
class InklingMoE(nn.Module):
def __init__(
self,
@@ -417,7 +482,12 @@ class InklingMoE(nn.Module):
layer_id
), f"layer {layer_id}: NVFP4 shared experts are not supported"
self.sink_experts = InklingSinkExperts(
sink_experts_cls = (
InklingSinkExpertsLinear
if get_current_vllm_config().lora_config is not None
else InklingSinkExperts
)
self.sink_experts = sink_experts_cls(
n_experts=n_shared,
d_model=config.hidden_size,
d_mlp=config.intermediate_size,