[Model] Add Inkling LoRA support [4/N] (#48884)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Bugen Zhao
Giancarlo Delfin
Isotr0py
Isotr0py
Jee Jee Li
Roger Wang
Yifan Qiao
Claude Fable 5
OpenAI Codex
parent
fe784ff22e
commit
f3e9497e92
@@ -6,7 +6,9 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.lora.utils import get_supported_lora_modules
|
||||
from vllm.models.inkling.nvidia import moe
|
||||
from vllm.models.inkling.nvidia.model import _TmlForCausalLMBase
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -62,6 +64,29 @@ def test_gate_uses_ll_bf16_gemm_through_token_limit(
|
||||
assert logits.dtype == torch.float32
|
||||
|
||||
|
||||
def test_gate_is_not_a_lora_target() -> None:
|
||||
model = torch.nn.Module()
|
||||
model.gate = moe.InklingGate(
|
||||
d_model=4,
|
||||
n_routed_experts=5,
|
||||
n_shared_experts=2,
|
||||
experts_per_token=2,
|
||||
route_scale=1.0,
|
||||
)
|
||||
|
||||
assert "gate" not in get_supported_lora_modules(model)
|
||||
|
||||
|
||||
def test_custom_embedding_is_not_a_lora_target() -> None:
|
||||
model = torch.nn.Module()
|
||||
model.embedding_modules = _TmlForCausalLMBase.embedding_modules
|
||||
|
||||
supported = get_supported_lora_modules(model)
|
||||
|
||||
assert "embed_tokens" not in supported
|
||||
assert "lm_head" in supported
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)])
|
||||
def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
|
||||
experts = SimpleNamespace(
|
||||
|
||||
+19
-1
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
import torch
|
||||
@@ -75,8 +76,14 @@ class LoRAConfig:
|
||||
"""If True, force the engine to use the universal 2D MoE LoRA wrapper
|
||||
(`FusedMoEWithLoRA`) regardless of the model's `is_3d_moe_weight` flag, so
|
||||
that 2D-format and 3D-format MoE LoRA adapters can be served in the same
|
||||
deployment. Only meaningful forMoE models; ignored otherwise. Default False
|
||||
deployment. Only meaningful for MoE models; ignored otherwise. Default False
|
||||
keeps the existing model-driven behavior."""
|
||||
enable_moe_shared_loras: bool = False
|
||||
"""If True, load MoE expert adapters in the "shared-outer" layout, where the
|
||||
gate/up (`w1`/`w3`) lora_A and the down (`w2`) lora_B are shared across all
|
||||
experts (stored once with expert-dim 1) instead of per-expert. The shared
|
||||
factors are broadcast to the expert count at kernel time. Only meaningful for
|
||||
MoE models whose adapters use this layout; ignored otherwise."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
@@ -97,6 +104,7 @@ class LoRAConfig:
|
||||
factors.append(self.lora_dtype)
|
||||
factors.append(self.enable_tower_connector_lora)
|
||||
factors.append(self.enable_mixed_moe_lora_format)
|
||||
factors.append(self.enable_moe_shared_loras)
|
||||
# target_modules affects which modules get LoRA applied
|
||||
factors.append(
|
||||
tuple(sorted(self.target_modules)) if self.target_modules else None
|
||||
@@ -129,3 +137,13 @@ class LoRAConfig:
|
||||
self.lora_dtype = model_config.dtype
|
||||
elif isinstance(self.lora_dtype, str):
|
||||
self.lora_dtype = getattr(torch, self.lora_dtype)
|
||||
|
||||
architectures = getattr(model_config, "architectures", None) or []
|
||||
is_inkling = any("Inkling" in arch for arch in architectures)
|
||||
if is_inkling and os.environ.get("INKLING_MULTIMEM_AR", "1") != "0":
|
||||
raise ValueError(
|
||||
"Inkling LoRA requires INKLING_MULTIMEM_AR=0: the Lamport "
|
||||
"fused-collective path bypasses the LoRA-wrapped wo_ud and dense "
|
||||
"down_proj layers on decode-sized batches, silently dropping "
|
||||
"their LoRA. Set INKLING_MULTIMEM_AR=0."
|
||||
)
|
||||
|
||||
@@ -602,6 +602,7 @@ class EngineArgs:
|
||||
enable_tower_connector_lora: bool = LoRAConfig.enable_tower_connector_lora
|
||||
specialize_active_lora: bool = LoRAConfig.specialize_active_lora
|
||||
enable_mixed_moe_lora_format: bool = LoRAConfig.enable_mixed_moe_lora_format
|
||||
enable_moe_shared_loras: bool = LoRAConfig.enable_moe_shared_loras
|
||||
|
||||
ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight
|
||||
num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override
|
||||
@@ -1351,6 +1352,10 @@ class EngineArgs:
|
||||
"--enable-mixed-moe-lora-format",
|
||||
**lora_kwargs["enable_mixed_moe_lora_format"],
|
||||
)
|
||||
lora_group.add_argument(
|
||||
"--enable-moe-shared-loras",
|
||||
**lora_kwargs["enable_moe_shared_loras"],
|
||||
)
|
||||
|
||||
# Observability arguments
|
||||
observability_kwargs = get_kwargs(ObservabilityConfig)
|
||||
@@ -2213,6 +2218,7 @@ class EngineArgs:
|
||||
enable_tower_connector_lora=self.enable_tower_connector_lora,
|
||||
specialize_active_lora=self.specialize_active_lora,
|
||||
enable_mixed_moe_lora_format=self.enable_mixed_moe_lora_format,
|
||||
enable_moe_shared_loras=self.enable_moe_shared_loras,
|
||||
max_cpu_loras=self.max_cpu_loras
|
||||
if self.max_cpu_loras and self.max_cpu_loras > 0
|
||||
else None,
|
||||
|
||||
@@ -57,6 +57,9 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
# For non-gated MoE (is_act_and_mul=False), only 1 slice is needed
|
||||
# since there's only up_proj (w1), not gate_proj + up_proj (w1 + w3)
|
||||
self._w13_slices = 2 if base_layer.moe_config.is_act_and_mul else 1
|
||||
# Set from lora_config.enable_moe_shared_loras in create_lora_weights.
|
||||
# When True, w13 lora_A and w2 lora_B are stored once for all experts.
|
||||
self.enable_moe_shared_loras = False
|
||||
# Mirrors per-(lora_id) layout of `self.lora_a_stacked` (built in
|
||||
# `create_lora_weights`) so `create_dummy_lora`'s n_slices fallback
|
||||
# matches `lora_a_stacked` length under EP.
|
||||
@@ -150,10 +153,21 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
local_num_experts=self.local_num_experts,
|
||||
punica_wrapper=self.punica_wrapper,
|
||||
use_tuned_config=bool(envs.VLLM_TUNED_CONFIG_FOLDER),
|
||||
enable_moe_shared_loras=self.enable_moe_shared_loras,
|
||||
aux_stream=self._lora_stream if use_dual_stream else None,
|
||||
events=self._events if use_dual_stream else None,
|
||||
)
|
||||
|
||||
@property
|
||||
def _w13_a_num_experts(self) -> int:
|
||||
"""Expert-dim of the w13 lora_A buffer: 1 when shared."""
|
||||
return 1 if self.enable_moe_shared_loras else self.local_num_experts
|
||||
|
||||
@property
|
||||
def _w2_b_num_experts(self) -> int:
|
||||
"""Expert-dim of the w2 lora_B buffer: 1 when shared."""
|
||||
return 1 if self.enable_moe_shared_loras else self.local_num_experts
|
||||
|
||||
def _create_lora_a_weights(
|
||||
self,
|
||||
max_loras: int,
|
||||
@@ -163,7 +177,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
torch.zeros(
|
||||
(
|
||||
max_loras,
|
||||
self.local_num_experts,
|
||||
self._w13_a_num_experts,
|
||||
lora_config.max_lora_rank
|
||||
if not self.fully_sharded
|
||||
else divide(lora_config.max_lora_rank, self.tp_size),
|
||||
@@ -205,7 +219,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
torch.zeros(
|
||||
(
|
||||
max_loras,
|
||||
self.local_num_experts,
|
||||
self._w2_b_num_experts,
|
||||
self.hidden_size
|
||||
if not self.fully_sharded
|
||||
else divide(self.hidden_size, self.tp_size),
|
||||
@@ -247,6 +261,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
self._verify_ep_fs(lora_config)
|
||||
self.max_loras = lora_config.max_loras
|
||||
self.fully_sharded = lora_config.fully_sharded_loras
|
||||
self.enable_moe_shared_loras = lora_config.enable_moe_shared_loras
|
||||
|
||||
self.adapter_enabled = torch.tensor(
|
||||
[0] * (max_loras + 1), dtype=torch.int, device=self.device
|
||||
@@ -263,8 +278,11 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
for experts_id in range(self.local_num_experts):
|
||||
# For gated MoE: gate_proj (w1), down_proj (w2), up_proj (w3)
|
||||
# For non-gated MoE: up_proj (w1), down_proj (w2)
|
||||
# Shared factors are collapsed (expert-dim 1): index 0.
|
||||
w13_a_eid = 0 if self.enable_moe_shared_loras else experts_id
|
||||
w2_b_eid = 0 if self.enable_moe_shared_loras else experts_id
|
||||
self.lora_a_stacked.append(
|
||||
self.w13_lora_a_stacked[0][lora_id][experts_id]
|
||||
self.w13_lora_a_stacked[0][lora_id][w13_a_eid]
|
||||
)
|
||||
self.lora_a_stacked.append(
|
||||
self.w2_lora_a_stacked[0][lora_id][experts_id]
|
||||
@@ -273,14 +291,12 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
self.lora_b_stacked.append(
|
||||
self.w13_lora_b_stacked[0][lora_id][experts_id]
|
||||
)
|
||||
self.lora_b_stacked.append(
|
||||
self.w2_lora_b_stacked[0][lora_id][experts_id]
|
||||
)
|
||||
self.lora_b_stacked.append(self.w2_lora_b_stacked[0][lora_id][w2_b_eid])
|
||||
|
||||
# Only add w3 (up_proj) for gated MoE (_w13_slices == 2)
|
||||
if self._w13_slices == 2:
|
||||
self.lora_a_stacked.append(
|
||||
self.w13_lora_a_stacked[1][lora_id][experts_id]
|
||||
self.w13_lora_a_stacked[1][lora_id][w13_a_eid]
|
||||
)
|
||||
self.lora_b_stacked.append(
|
||||
self.w13_lora_b_stacked[1][lora_id][experts_id]
|
||||
@@ -340,6 +356,22 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
|
||||
return w2_lora_b[:, start_idx:end_idx, :]
|
||||
|
||||
def _match_expert_dim(
|
||||
self, src: torch.Tensor, buffer: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Align a LoRA factor's expert-dim (dim 0) to its stacked buffer.
|
||||
|
||||
Equal dims pass through. When the buffer is collapsed to expert-dim 1
|
||||
(a shared factor) but the source carries per-expert copies, keep
|
||||
the first — every expert shares the same factor, so the copies are
|
||||
identical for a real adapter and irrelevant (zeros) for the dummy.
|
||||
"""
|
||||
tgt = buffer.shape[1]
|
||||
if src.shape[0] == tgt:
|
||||
return src
|
||||
assert tgt == 1, f"expert-dim mismatch: source {src.shape[0]} vs buffer {tgt}"
|
||||
return src[:1]
|
||||
|
||||
def reset_lora(self, index: int):
|
||||
"""Resets the lora weights at index back to 0."""
|
||||
for pos in range(self._w13_slices):
|
||||
@@ -366,20 +398,24 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
self.reset_lora(index)
|
||||
self.adapter_enabled[index] = 1
|
||||
|
||||
num_experts = self.w13_lora_a_stacked[0].shape[1]
|
||||
|
||||
w1_lora_a, w2_lora_a, w3_lora_a = lora_a
|
||||
w1_lora_b, w2_lora_b, w3_lora_b = lora_b
|
||||
|
||||
# EP slicing is done once at add time in
|
||||
# LoRAModelManager._slice_moe_lora_ep, so by here the cached
|
||||
# tensors already match the local-expert dim of the stacked buffers.
|
||||
assert (
|
||||
num_experts
|
||||
== w1_lora_a.shape[0]
|
||||
== w2_lora_a.shape[0]
|
||||
== w3_lora_a.shape[0]
|
||||
)
|
||||
# For shared-loras adapters the w13 lora_A and w2 lora_B buffers are
|
||||
# collapsed to expert-dim 1: match each factor to the expert-dim of
|
||||
# the buffer it is copied into. A real shared factor already has
|
||||
# expert-dim 1 (no-op); a per-expert source (e.g. the warmup dummy
|
||||
# LoRA, which always stacks to num_experts) is collapsed to the shared
|
||||
# slot since every expert shares the same factor.
|
||||
w1_lora_a = self._match_expert_dim(w1_lora_a, self.w13_lora_a_stacked[0])
|
||||
w3_lora_a = self._match_expert_dim(w3_lora_a, self.w13_lora_a_stacked[-1])
|
||||
w2_lora_a = self._match_expert_dim(w2_lora_a, self.w2_lora_a_stacked[0])
|
||||
w1_lora_b = self._match_expert_dim(w1_lora_b, self.w13_lora_b_stacked[0])
|
||||
w3_lora_b = self._match_expert_dim(w3_lora_b, self.w13_lora_b_stacked[-1])
|
||||
w2_lora_b = self._match_expert_dim(w2_lora_b, self.w2_lora_b_stacked[0])
|
||||
|
||||
slliced_w1_lora_a = self._slice_w13_a(w1_lora_a)
|
||||
slliced_w1_lora_b = self._slice_w13_b(w1_lora_b)
|
||||
|
||||
@@ -227,6 +227,39 @@ class PackedLoRALayerWeights(LoRALayerWeights):
|
||||
)
|
||||
return obj
|
||||
|
||||
@classmethod
|
||||
def pack_moe_stacked(
|
||||
cls,
|
||||
loras: GenericSequence["LoRALayerWeights | None"],
|
||||
module_name: str,
|
||||
) -> "PackedLoRALayerWeights":
|
||||
"""Pack pre-stacked (3D) w1/w2/w3 expert LoRAs into a single LoRA.
|
||||
|
||||
Unlike :meth:`pack_moe`, which stacks one 2D tensor per expert, this
|
||||
expects each of the three input LoRAs to already carry the expert
|
||||
dimension (``experts.w{1,2,3}``). It is used for "shared-outer"
|
||||
adapters where some factors are shared across experts (expert-dim 1)
|
||||
and cannot be reconstructed by stacking per-expert tensors. The
|
||||
produced ``lora_a``/``lora_b`` are the ``[w1, w2, w3]`` lists that
|
||||
``FusedMoEWithLoRA.set_lora`` consumes directly.
|
||||
"""
|
||||
assert len(loras) == 3, (
|
||||
"shared-outer expert LoRA expects exactly w1/w2/w3 stacked tensors"
|
||||
)
|
||||
w1_lora, w2_lora, w3_lora = loras
|
||||
assert w1_lora is not None and w2_lora is not None and w3_lora is not None
|
||||
rank = w1_lora.rank
|
||||
lora_alpha = w1_lora.lora_alpha
|
||||
scaling = lora_alpha / rank
|
||||
return cls(
|
||||
module_name,
|
||||
rank,
|
||||
[lora_alpha, lora_alpha, lora_alpha],
|
||||
[w1_lora.lora_a, w2_lora.lora_a, w3_lora.lora_a],
|
||||
[w1_lora.lora_b, w2_lora.lora_b, w3_lora.lora_b],
|
||||
scaling=[scaling, scaling, scaling],
|
||||
)
|
||||
|
||||
def optimize(self) -> "PackedLoRALayerWeights":
|
||||
"""Optimize the LoRA by merging the scaling into lora_b."""
|
||||
for i in range(len(self.lora_b)):
|
||||
|
||||
+26
-24
@@ -132,8 +132,13 @@ class LoRAModelManager:
|
||||
and self.model.is_3d_moe_weight
|
||||
and not self._enable_mixed_moe_lora_format
|
||||
)
|
||||
# Shared MoE adapters: w13 lora_A / w2 lora_B shared across experts,
|
||||
# stored as pre-stacked experts.w{1,2,3} tensors (startup opt-in).
|
||||
self._enable_moe_shared_loras = is_moe and lora_config.enable_moe_shared_loras
|
||||
self.packed_modules_mapping = process_packed_modules_mapping(
|
||||
self.model, force_2d_moe=self._enable_mixed_moe_lora_format
|
||||
self.model,
|
||||
force_2d_moe=self._enable_mixed_moe_lora_format,
|
||||
enable_moe_shared_loras=self._enable_moe_shared_loras,
|
||||
)
|
||||
self._is_non_gated_moe = is_moe and self.model.is_non_gated_moe
|
||||
self._use_ep = bool(
|
||||
@@ -761,6 +766,16 @@ class LoRAModelManager:
|
||||
if lora_model.check_lora_name(replaced_module_name):
|
||||
module_name = replaced_module_name
|
||||
if module_name.endswith(".experts"):
|
||||
if self._enable_moe_shared_loras:
|
||||
lora_model.loras[module_name] = (
|
||||
PackedLoRALayerWeights.pack_moe_stacked(
|
||||
replacement_loras,
|
||||
module_name,
|
||||
)
|
||||
)
|
||||
for module in packed_module_names:
|
||||
lora_model.loras.pop(module, None)
|
||||
continue
|
||||
if self._is_non_gated_moe and len(replacement_loras) > 0:
|
||||
replacement_loras = self._pad_lora_pairs_to_triplets(
|
||||
replacement_loras
|
||||
@@ -1019,20 +1034,10 @@ class LoRAModelManager:
|
||||
module: FusedMoEWithLoRA,
|
||||
module_name: str,
|
||||
) -> None:
|
||||
"""Slice the cached LoRA tensors down to this rank's local experts.
|
||||
"""Slice cached LoRA tensors down to this rank's local experts.
|
||||
|
||||
The 2D MoE checkpoint enters as a list of per-(w1/w2/w3) tensors of
|
||||
shape (num_experts, rank, in) / (num_experts, out, rank). When EP
|
||||
is active each rank only owns local_num_experts; without this slice
|
||||
the CPU LoRAModel keeps the full global weight and set_lora has to
|
||||
re-slice on every activation.
|
||||
|
||||
With the load-time / pack-time slicing in
|
||||
``_restrict_to_local_experts``, the stacked tensors already match
|
||||
``local_num_experts`` and the inner branch becomes a no-op. The
|
||||
guard remains so checkpoints that bypassed the pre-slicing (e.g.
|
||||
``.bin``/``.pt`` adapters with weights mappers we don't recognize)
|
||||
still get sliced here.
|
||||
Shared factors have expert dimension one and remain available on every
|
||||
rank. Per-expert factors are narrowed to the local expert block.
|
||||
"""
|
||||
if not module.use_ep:
|
||||
return
|
||||
@@ -1046,16 +1051,13 @@ class LoRAModelManager:
|
||||
expert_start = ep_rank * local_num_experts
|
||||
expert_end = expert_start + local_num_experts
|
||||
|
||||
new_lora_a: list[torch.Tensor | None] = []
|
||||
new_lora_b: list[torch.Tensor | None] = []
|
||||
for a, b in zip(module_lora.lora_a, module_lora.lora_b):
|
||||
if a is not None and b is not None and a.shape[0] == global_num_experts:
|
||||
a = a[expert_start:expert_end].contiguous()
|
||||
b = b[expert_start:expert_end].contiguous()
|
||||
new_lora_a.append(a)
|
||||
new_lora_b.append(b)
|
||||
module_lora.lora_a = new_lora_a
|
||||
module_lora.lora_b = new_lora_b
|
||||
def _slice_local(t: torch.Tensor | None) -> torch.Tensor | None:
|
||||
if t is not None and t.shape[0] == global_num_experts:
|
||||
return t[expert_start:expert_end].contiguous()
|
||||
return t
|
||||
|
||||
module_lora.lora_a = [_slice_local(a) for a in module_lora.lora_a]
|
||||
module_lora.lora_b = [_slice_local(b) for b in module_lora.lora_b]
|
||||
|
||||
def _restrict_to_local_experts(
|
||||
self, module_name: str, new_module_names: list[str]
|
||||
|
||||
+14
-2
@@ -369,7 +369,9 @@ def get_adapter_absolute_path(lora_path: str) -> str:
|
||||
|
||||
|
||||
def process_packed_modules_mapping(
|
||||
model: nn.Module, force_2d_moe: bool = False
|
||||
model: nn.Module,
|
||||
force_2d_moe: bool = False,
|
||||
enable_moe_shared_loras: bool = False,
|
||||
) -> dict[str, list[str]]:
|
||||
if is_moe_model(model):
|
||||
# This method generates and returns a dictionary mapping packed module
|
||||
@@ -382,7 +384,17 @@ def process_packed_modules_mapping(
|
||||
# the engine forces the universal 2D wrapper via
|
||||
# enable_mixed_moe_lora_format (so 3D models can also load 2D
|
||||
# adapters through FusedMoEWithLoRA).
|
||||
if (not model.is_3d_moe_weight) or force_2d_moe:
|
||||
if enable_moe_shared_loras:
|
||||
# Shared MoE adapters store one pre-stacked tensor per
|
||||
# expert-projection (experts.w1/w2/w3) rather than a per-expert
|
||||
# tensor list, so the packed mapping references the three stack
|
||||
# names directly (drives expected_lora_modules and the packing).
|
||||
packed_modules_mapping["experts"] = [
|
||||
"experts.w1",
|
||||
"experts.w2",
|
||||
"experts.w3",
|
||||
]
|
||||
elif (not model.is_3d_moe_weight) or force_2d_moe:
|
||||
# Filter out malformed entries: non-gated MoE has empty
|
||||
# ckpt_up_proj_name which results in weight_name containing ".."
|
||||
# (e.g., "experts.0.." instead of "experts.0.layer_name.")
|
||||
|
||||
@@ -43,6 +43,11 @@ class MoELoRAContext:
|
||||
# try_get_optimal_moe_lora_config for Triton kernel tile configs.
|
||||
use_tuned_config: bool
|
||||
|
||||
# Shared MoE LoRA: w13 lora_A and w2 lora_B are shared across all experts
|
||||
# (stored collapsed with expert-dim 1). When True, LoRAExpertsMixin expands
|
||||
# them to local_num_experts via a stride-0 view before the kernel.
|
||||
enable_moe_shared_loras: bool = False
|
||||
|
||||
# Optional dual-stream support for overlapping each (base GEMM, LoRA)
|
||||
# pair. When aux_stream is None, the experts.apply() path runs the
|
||||
# original sequential schedule. When set, base GEMM runs on the default
|
||||
|
||||
@@ -52,10 +52,19 @@ class LoRAExpertsMixin:
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
]:
|
||||
w13_lora_a_stacked = lora_context.w13_lora_a_stacked
|
||||
if lora_context.enable_moe_shared_loras:
|
||||
# w13 lora_A is shared across experts (collapsed expert-dim 1);
|
||||
# broadcast to local_num_experts via a stride-0 view. The kernel
|
||||
# derives num_experts from shape[1] and indexes via stride(1)==0.
|
||||
w13_lora_a_stacked = tuple(
|
||||
a.expand(-1, lora_context.local_num_experts, -1, -1)
|
||||
for a in w13_lora_a_stacked
|
||||
)
|
||||
return lora_context.punica_wrapper.add_lora_w13(
|
||||
y,
|
||||
x,
|
||||
lora_context.w13_lora_a_stacked,
|
||||
w13_lora_a_stacked,
|
||||
lora_context.w13_lora_b_stacked,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
@@ -92,11 +101,19 @@ class LoRAExpertsMixin:
|
||||
top_k_num: int,
|
||||
add_inputs: bool = True,
|
||||
) -> None:
|
||||
w2_lora_b_stacked = lora_context.w2_lora_b_stacked
|
||||
if lora_context.enable_moe_shared_loras:
|
||||
# w2 lora_B is shared across experts (collapsed expert-dim 1);
|
||||
# broadcast to local_num_experts via a stride-0 view.
|
||||
w2_lora_b_stacked = tuple(
|
||||
b.expand(-1, lora_context.local_num_experts, -1, -1)
|
||||
for b in w2_lora_b_stacked
|
||||
)
|
||||
lora_context.punica_wrapper.add_lora_w2(
|
||||
y,
|
||||
x,
|
||||
lora_context.w2_lora_a_stacked,
|
||||
lora_context.w2_lora_b_stacked,
|
||||
w2_lora_b_stacked,
|
||||
topk_weights,
|
||||
sorted_token_ids_lora,
|
||||
expert_ids_lora,
|
||||
|
||||
@@ -1,6 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling logits processor with muP output scaling."""
|
||||
"""Inkling logits processor (muP + LoRA aware).
|
||||
|
||||
Inkling divides the final logits by a muP width multiplier
|
||||
(``logits_mup_width_multiplier``). This applies it two ways, depending on
|
||||
whether an lm_head LoRA is attached:
|
||||
|
||||
* No LoRA: fold ``1/mup`` into the lm_head GEMM alpha (fp32 epilogue) -- no
|
||||
separate elementwise kernel, no extra rounding, no weight mutation.
|
||||
* LoRA attached: the LoRA manager wraps this layer in
|
||||
``LogitsProcessorWithLoRA``, whose ``forward`` calls
|
||||
``type(base_layer).forward(self=wrapper)`` -- so this ``forward`` runs with
|
||||
``self`` bound to the wrapper. We detect that via ``base_layer`` and take the
|
||||
LoRA path: run the wrapper's ``_get_logits`` (base logits + the lm_head LoRA
|
||||
delta), then divide the full logits by the multiplier so the delta is scaled
|
||||
too. muP thus composes with the LoRA delta, with the dispatch as the only
|
||||
model-side branching.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -47,6 +63,46 @@ class InklingLogitsProcessor(LogitsProcessor):
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | None:
|
||||
# ``base_layer`` exists only on the LogitsProcessorWithLoRA wrapper,
|
||||
# which calls this forward with ``self`` bound to the wrapper. The
|
||||
# wrapper is not an ``InklingLogitsProcessor`` instance, so dispatch
|
||||
# ``_lora_forward`` explicitly through the base_layer's class (it
|
||||
# provides ``_get_logits``/``logits_as_input``; only ``_lora_forward``
|
||||
# lives on this class).
|
||||
if hasattr(self, "base_layer"):
|
||||
return type(self.base_layer)._lora_forward(
|
||||
self, lm_head, hidden_states, embedding_bias
|
||||
)
|
||||
return self._base_forward(lm_head, hidden_states, embedding_bias)
|
||||
|
||||
def _lora_forward(
|
||||
self,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | None:
|
||||
# ``self`` is the LogitsProcessorWithLoRA wrapper here: ``_get_logits``
|
||||
# returns the base logits plus the lm_head LoRA delta. Apply the muP
|
||||
# divisor on the full logits so the LoRA delta is scaled too.
|
||||
mup_multiplier = self.base_layer.logits_mup_width_multiplier
|
||||
mup = 1.0 / mup_multiplier if mup_multiplier else None
|
||||
if self.logits_as_input:
|
||||
logits = hidden_states
|
||||
else:
|
||||
logits = self._get_logits(hidden_states, lm_head, embedding_bias)
|
||||
# TODO: fuse this multiplication
|
||||
if logits is not None and mup:
|
||||
assert self.base_layer.soft_cap is None
|
||||
assert self.base_layer.scale == 1.0
|
||||
logits = logits * mup
|
||||
return logits
|
||||
|
||||
def _base_forward(
|
||||
self,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | None:
|
||||
mup = self.logits_mup_width_multiplier
|
||||
if not mup:
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsLoRA,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
@@ -360,7 +361,7 @@ class InklingModel(nn.Module):
|
||||
return self.norm(hidden_states)
|
||||
|
||||
|
||||
class _TmlForCausalLMBase(nn.Module, SupportsPP):
|
||||
class _TmlForCausalLMBase(nn.Module, SupportsPP, SupportsLoRA):
|
||||
"""Shared text-backbone causal-LM scaffolding for both entry classes."""
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
@@ -380,6 +381,8 @@ class _TmlForCausalLMBase(nn.Module, SupportsPP):
|
||||
"model.llm.embed": "model.embed_tokens",
|
||||
"model.llm.norm": "model.norm",
|
||||
"model.llm.unembed": "lm_head",
|
||||
"language_model.layers.": "model.layers.",
|
||||
"language_model.lm_head.": "lm_head.",
|
||||
},
|
||||
orig_to_new_suffix={
|
||||
# NVFP4 scale
|
||||
@@ -389,6 +392,13 @@ class _TmlForCausalLMBase(nn.Module, SupportsPP):
|
||||
".w2_weight.scale2": ".w2_weight_scale_2",
|
||||
},
|
||||
)
|
||||
packed_modules_mapping = {
|
||||
"qkvr": ["wq_du", "wk_dv", "wv_dv", "wr_du"],
|
||||
"w13": ["w1", "w3"],
|
||||
}
|
||||
embedding_modules = {
|
||||
"lm_head": "output_embeddings",
|
||||
}
|
||||
|
||||
def _build(
|
||||
self,
|
||||
|
||||
@@ -343,6 +343,71 @@ class InklingSinkExperts(nn.Module):
|
||||
return h @ self.w2_weight.T # (T, D)
|
||||
|
||||
|
||||
class InklingSinkExpertsLinear(nn.Module):
|
||||
"""LoRA-capable implementation of the Inkling sink experts."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
n_experts: int,
|
||||
d_model: int,
|
||||
d_mlp: int,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
|
||||
self.n_experts = n_experts
|
||||
self.d_mlp = d_mlp
|
||||
total = n_experts * d_mlp
|
||||
self.w13 = MergedColumnParallelLinear(
|
||||
input_size=d_model,
|
||||
output_sizes=[total, total],
|
||||
bias=False,
|
||||
prefix=f"{prefix}.w13",
|
||||
)
|
||||
self.w2 = RowParallelLinear(
|
||||
input_size=total,
|
||||
output_size=d_model,
|
||||
bias=False,
|
||||
reduce_results=False,
|
||||
prefix=f"{prefix}.w2",
|
||||
)
|
||||
self._w2_input_pp = self.w2.input_size_per_partition
|
||||
self._col_expert: torch.Tensor | None = None
|
||||
|
||||
def _gamma_expand(self, gammas: torch.Tensor) -> torch.Tensor:
|
||||
if self._col_expert is None or self._col_expert.device != gammas.device:
|
||||
local = self._w2_input_pp
|
||||
start = get_tensor_model_parallel_rank() * local
|
||||
cols = torch.arange(start, start + local, device=gammas.device)
|
||||
self._col_expert = (cols // self.d_mlp).long()
|
||||
return gammas[:, self._col_expert]
|
||||
|
||||
def load_weight(self, key: str, weight: torch.Tensor) -> list[str]:
|
||||
if key == "w13_weight":
|
||||
d_model = weight.shape[-1]
|
||||
gate = weight[:, 0::2, :].reshape(-1, d_model).contiguous()
|
||||
up = weight[:, 1::2, :].reshape(-1, d_model).contiguous()
|
||||
self.w13.weight_loader(self.w13.weight, gate, 0)
|
||||
self.w13.weight_loader(self.w13.weight, up, 1)
|
||||
return ["w13.weight"]
|
||||
w = weight.permute(1, 0, 2).reshape(weight.shape[1], -1).contiguous()
|
||||
self.w2.weight_loader(self.w2.weight, w)
|
||||
return ["w2.weight"]
|
||||
|
||||
def forward(self, x: torch.Tensor, gammas: torch.Tensor) -> torch.Tensor:
|
||||
gate_up, _ = self.w13(x)
|
||||
gate, up = gate_up.chunk(2, dim=-1)
|
||||
hidden_states = torch.nn.functional.silu(gate) * up
|
||||
hidden_states = (hidden_states * self._gamma_expand(gammas)).to(x.dtype)
|
||||
output, _ = self.w2(hidden_states)
|
||||
return output
|
||||
|
||||
|
||||
class InklingMoE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -417,7 +482,12 @@ class InklingMoE(nn.Module):
|
||||
layer_id
|
||||
), f"layer {layer_id}: NVFP4 shared experts are not supported"
|
||||
|
||||
self.sink_experts = InklingSinkExperts(
|
||||
sink_experts_cls = (
|
||||
InklingSinkExpertsLinear
|
||||
if get_current_vllm_config().lora_config is not None
|
||||
else InklingSinkExperts
|
||||
)
|
||||
self.sink_experts = sink_experts_cls(
|
||||
n_experts=n_shared,
|
||||
d_model=config.hidden_size,
|
||||
d_mlp=config.intermediate_size,
|
||||
|
||||
Reference in New Issue
Block a user