From f3e9497e921a16741401c5e93af0c2c29ea74907 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Thu, 16 Jul 2026 18:52:15 -0700 Subject: [PATCH] [Model] Add Inkling LoRA support [4/N] (#48884) Signed-off-by: Woosuk Kwon Co-authored-by: Bugen Zhao Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Isotr0py Co-authored-by: Isotr0py Co-authored-by: Jee Jee Li Co-authored-by: Roger Wang Co-authored-by: Yifan Qiao Co-authored-by: Claude Fable 5 Co-authored-by: OpenAI Codex --- .../models/inkling/test_moe_weight_layout.py | 25 +++++++ vllm/config/lora.py | 20 +++++- vllm/engine/arg_utils.py | 6 ++ vllm/lora/layers/fused_moe.py | 66 +++++++++++++---- vllm/lora/lora_weights.py | 33 +++++++++ vllm/lora/model_manager.py | 50 ++++++------- vllm/lora/utils.py | 16 ++++- .../layers/fused_moe/experts/lora_context.py | 5 ++ .../fused_moe/experts/lora_experts_mixin.py | 21 +++++- .../models/inkling/nvidia/logits_processor.py | 58 ++++++++++++++- vllm/models/inkling/nvidia/model.py | 12 +++- vllm/models/inkling/nvidia/moe.py | 72 ++++++++++++++++++- 12 files changed, 337 insertions(+), 47 deletions(-) diff --git a/tests/models/inkling/test_moe_weight_layout.py b/tests/models/inkling/test_moe_weight_layout.py index e46db9be32..e6c35c57b4 100644 --- a/tests/models/inkling/test_moe_weight_layout.py +++ b/tests/models/inkling/test_moe_weight_layout.py @@ -6,7 +6,9 @@ from types import SimpleNamespace import pytest import torch +from vllm.lora.utils import get_supported_lora_modules from vllm.models.inkling.nvidia import moe +from vllm.models.inkling.nvidia.model import _TmlForCausalLMBase from vllm.platforms import current_platform @@ -62,6 +64,29 @@ def test_gate_uses_ll_bf16_gemm_through_token_limit( assert logits.dtype == torch.float32 +def test_gate_is_not_a_lora_target() -> None: + model = torch.nn.Module() + model.gate = moe.InklingGate( + d_model=4, + n_routed_experts=5, + n_shared_experts=2, + experts_per_token=2, + route_scale=1.0, + ) + + assert "gate" not in get_supported_lora_modules(model) + + +def test_custom_embedding_is_not_a_lora_target() -> None: + model = torch.nn.Module() + model.embedding_modules = _TmlForCausalLMBase.embedding_modules + + supported = get_supported_lora_modules(model) + + assert "embed_tokens" not in supported + assert "lm_head" in supported + + @pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)]) def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None: experts = SimpleNamespace( diff --git a/vllm/config/lora.py b/vllm/config/lora.py index 94a679941c..4323da96d8 100644 --- a/vllm/config/lora.py +++ b/vllm/config/lora.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import os from typing import TYPE_CHECKING, Any, Literal import torch @@ -75,8 +76,14 @@ class LoRAConfig: """If True, force the engine to use the universal 2D MoE LoRA wrapper (`FusedMoEWithLoRA`) regardless of the model's `is_3d_moe_weight` flag, so that 2D-format and 3D-format MoE LoRA adapters can be served in the same - deployment. Only meaningful forMoE models; ignored otherwise. Default False + deployment. Only meaningful for MoE models; ignored otherwise. Default False keeps the existing model-driven behavior.""" + enable_moe_shared_loras: bool = False + """If True, load MoE expert adapters in the "shared-outer" layout, where the + gate/up (`w1`/`w3`) lora_A and the down (`w2`) lora_B are shared across all + experts (stored once with expert-dim 1) instead of per-expert. The shared + factors are broadcast to the expert count at kernel time. Only meaningful for + MoE models whose adapters use this layout; ignored otherwise.""" def compute_hash(self) -> str: """ @@ -97,6 +104,7 @@ class LoRAConfig: factors.append(self.lora_dtype) factors.append(self.enable_tower_connector_lora) factors.append(self.enable_mixed_moe_lora_format) + factors.append(self.enable_moe_shared_loras) # target_modules affects which modules get LoRA applied factors.append( tuple(sorted(self.target_modules)) if self.target_modules else None @@ -129,3 +137,13 @@ class LoRAConfig: self.lora_dtype = model_config.dtype elif isinstance(self.lora_dtype, str): self.lora_dtype = getattr(torch, self.lora_dtype) + + architectures = getattr(model_config, "architectures", None) or [] + is_inkling = any("Inkling" in arch for arch in architectures) + if is_inkling and os.environ.get("INKLING_MULTIMEM_AR", "1") != "0": + raise ValueError( + "Inkling LoRA requires INKLING_MULTIMEM_AR=0: the Lamport " + "fused-collective path bypasses the LoRA-wrapped wo_ud and dense " + "down_proj layers on decode-sized batches, silently dropping " + "their LoRA. Set INKLING_MULTIMEM_AR=0." + ) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 366f6eca68..72dfc28688 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -602,6 +602,7 @@ class EngineArgs: enable_tower_connector_lora: bool = LoRAConfig.enable_tower_connector_lora specialize_active_lora: bool = LoRAConfig.specialize_active_lora enable_mixed_moe_lora_format: bool = LoRAConfig.enable_mixed_moe_lora_format + enable_moe_shared_loras: bool = LoRAConfig.enable_moe_shared_loras ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override @@ -1351,6 +1352,10 @@ class EngineArgs: "--enable-mixed-moe-lora-format", **lora_kwargs["enable_mixed_moe_lora_format"], ) + lora_group.add_argument( + "--enable-moe-shared-loras", + **lora_kwargs["enable_moe_shared_loras"], + ) # Observability arguments observability_kwargs = get_kwargs(ObservabilityConfig) @@ -2213,6 +2218,7 @@ class EngineArgs: enable_tower_connector_lora=self.enable_tower_connector_lora, specialize_active_lora=self.specialize_active_lora, enable_mixed_moe_lora_format=self.enable_mixed_moe_lora_format, + enable_moe_shared_loras=self.enable_moe_shared_loras, max_cpu_loras=self.max_cpu_loras if self.max_cpu_loras and self.max_cpu_loras > 0 else None, diff --git a/vllm/lora/layers/fused_moe.py b/vllm/lora/layers/fused_moe.py index 63a4ea9a82..b447af8780 100644 --- a/vllm/lora/layers/fused_moe.py +++ b/vllm/lora/layers/fused_moe.py @@ -57,6 +57,9 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): # For non-gated MoE (is_act_and_mul=False), only 1 slice is needed # since there's only up_proj (w1), not gate_proj + up_proj (w1 + w3) self._w13_slices = 2 if base_layer.moe_config.is_act_and_mul else 1 + # Set from lora_config.enable_moe_shared_loras in create_lora_weights. + # When True, w13 lora_A and w2 lora_B are stored once for all experts. + self.enable_moe_shared_loras = False # Mirrors per-(lora_id) layout of `self.lora_a_stacked` (built in # `create_lora_weights`) so `create_dummy_lora`'s n_slices fallback # matches `lora_a_stacked` length under EP. @@ -150,10 +153,21 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): local_num_experts=self.local_num_experts, punica_wrapper=self.punica_wrapper, use_tuned_config=bool(envs.VLLM_TUNED_CONFIG_FOLDER), + enable_moe_shared_loras=self.enable_moe_shared_loras, aux_stream=self._lora_stream if use_dual_stream else None, events=self._events if use_dual_stream else None, ) + @property + def _w13_a_num_experts(self) -> int: + """Expert-dim of the w13 lora_A buffer: 1 when shared.""" + return 1 if self.enable_moe_shared_loras else self.local_num_experts + + @property + def _w2_b_num_experts(self) -> int: + """Expert-dim of the w2 lora_B buffer: 1 when shared.""" + return 1 if self.enable_moe_shared_loras else self.local_num_experts + def _create_lora_a_weights( self, max_loras: int, @@ -163,7 +177,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): torch.zeros( ( max_loras, - self.local_num_experts, + self._w13_a_num_experts, lora_config.max_lora_rank if not self.fully_sharded else divide(lora_config.max_lora_rank, self.tp_size), @@ -205,7 +219,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): torch.zeros( ( max_loras, - self.local_num_experts, + self._w2_b_num_experts, self.hidden_size if not self.fully_sharded else divide(self.hidden_size, self.tp_size), @@ -247,6 +261,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): self._verify_ep_fs(lora_config) self.max_loras = lora_config.max_loras self.fully_sharded = lora_config.fully_sharded_loras + self.enable_moe_shared_loras = lora_config.enable_moe_shared_loras self.adapter_enabled = torch.tensor( [0] * (max_loras + 1), dtype=torch.int, device=self.device @@ -263,8 +278,11 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): for experts_id in range(self.local_num_experts): # For gated MoE: gate_proj (w1), down_proj (w2), up_proj (w3) # For non-gated MoE: up_proj (w1), down_proj (w2) + # Shared factors are collapsed (expert-dim 1): index 0. + w13_a_eid = 0 if self.enable_moe_shared_loras else experts_id + w2_b_eid = 0 if self.enable_moe_shared_loras else experts_id self.lora_a_stacked.append( - self.w13_lora_a_stacked[0][lora_id][experts_id] + self.w13_lora_a_stacked[0][lora_id][w13_a_eid] ) self.lora_a_stacked.append( self.w2_lora_a_stacked[0][lora_id][experts_id] @@ -273,14 +291,12 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): self.lora_b_stacked.append( self.w13_lora_b_stacked[0][lora_id][experts_id] ) - self.lora_b_stacked.append( - self.w2_lora_b_stacked[0][lora_id][experts_id] - ) + self.lora_b_stacked.append(self.w2_lora_b_stacked[0][lora_id][w2_b_eid]) # Only add w3 (up_proj) for gated MoE (_w13_slices == 2) if self._w13_slices == 2: self.lora_a_stacked.append( - self.w13_lora_a_stacked[1][lora_id][experts_id] + self.w13_lora_a_stacked[1][lora_id][w13_a_eid] ) self.lora_b_stacked.append( self.w13_lora_b_stacked[1][lora_id][experts_id] @@ -340,6 +356,22 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): return w2_lora_b[:, start_idx:end_idx, :] + def _match_expert_dim( + self, src: torch.Tensor, buffer: torch.Tensor + ) -> torch.Tensor: + """Align a LoRA factor's expert-dim (dim 0) to its stacked buffer. + + Equal dims pass through. When the buffer is collapsed to expert-dim 1 + (a shared factor) but the source carries per-expert copies, keep + the first — every expert shares the same factor, so the copies are + identical for a real adapter and irrelevant (zeros) for the dummy. + """ + tgt = buffer.shape[1] + if src.shape[0] == tgt: + return src + assert tgt == 1, f"expert-dim mismatch: source {src.shape[0]} vs buffer {tgt}" + return src[:1] + def reset_lora(self, index: int): """Resets the lora weights at index back to 0.""" for pos in range(self._w13_slices): @@ -366,20 +398,24 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): self.reset_lora(index) self.adapter_enabled[index] = 1 - num_experts = self.w13_lora_a_stacked[0].shape[1] - w1_lora_a, w2_lora_a, w3_lora_a = lora_a w1_lora_b, w2_lora_b, w3_lora_b = lora_b # EP slicing is done once at add time in # LoRAModelManager._slice_moe_lora_ep, so by here the cached # tensors already match the local-expert dim of the stacked buffers. - assert ( - num_experts - == w1_lora_a.shape[0] - == w2_lora_a.shape[0] - == w3_lora_a.shape[0] - ) + # For shared-loras adapters the w13 lora_A and w2 lora_B buffers are + # collapsed to expert-dim 1: match each factor to the expert-dim of + # the buffer it is copied into. A real shared factor already has + # expert-dim 1 (no-op); a per-expert source (e.g. the warmup dummy + # LoRA, which always stacks to num_experts) is collapsed to the shared + # slot since every expert shares the same factor. + w1_lora_a = self._match_expert_dim(w1_lora_a, self.w13_lora_a_stacked[0]) + w3_lora_a = self._match_expert_dim(w3_lora_a, self.w13_lora_a_stacked[-1]) + w2_lora_a = self._match_expert_dim(w2_lora_a, self.w2_lora_a_stacked[0]) + w1_lora_b = self._match_expert_dim(w1_lora_b, self.w13_lora_b_stacked[0]) + w3_lora_b = self._match_expert_dim(w3_lora_b, self.w13_lora_b_stacked[-1]) + w2_lora_b = self._match_expert_dim(w2_lora_b, self.w2_lora_b_stacked[0]) slliced_w1_lora_a = self._slice_w13_a(w1_lora_a) slliced_w1_lora_b = self._slice_w13_b(w1_lora_b) diff --git a/vllm/lora/lora_weights.py b/vllm/lora/lora_weights.py index f90724c5eb..38a5875525 100644 --- a/vllm/lora/lora_weights.py +++ b/vllm/lora/lora_weights.py @@ -227,6 +227,39 @@ class PackedLoRALayerWeights(LoRALayerWeights): ) return obj + @classmethod + def pack_moe_stacked( + cls, + loras: GenericSequence["LoRALayerWeights | None"], + module_name: str, + ) -> "PackedLoRALayerWeights": + """Pack pre-stacked (3D) w1/w2/w3 expert LoRAs into a single LoRA. + + Unlike :meth:`pack_moe`, which stacks one 2D tensor per expert, this + expects each of the three input LoRAs to already carry the expert + dimension (``experts.w{1,2,3}``). It is used for "shared-outer" + adapters where some factors are shared across experts (expert-dim 1) + and cannot be reconstructed by stacking per-expert tensors. The + produced ``lora_a``/``lora_b`` are the ``[w1, w2, w3]`` lists that + ``FusedMoEWithLoRA.set_lora`` consumes directly. + """ + assert len(loras) == 3, ( + "shared-outer expert LoRA expects exactly w1/w2/w3 stacked tensors" + ) + w1_lora, w2_lora, w3_lora = loras + assert w1_lora is not None and w2_lora is not None and w3_lora is not None + rank = w1_lora.rank + lora_alpha = w1_lora.lora_alpha + scaling = lora_alpha / rank + return cls( + module_name, + rank, + [lora_alpha, lora_alpha, lora_alpha], + [w1_lora.lora_a, w2_lora.lora_a, w3_lora.lora_a], + [w1_lora.lora_b, w2_lora.lora_b, w3_lora.lora_b], + scaling=[scaling, scaling, scaling], + ) + def optimize(self) -> "PackedLoRALayerWeights": """Optimize the LoRA by merging the scaling into lora_b.""" for i in range(len(self.lora_b)): diff --git a/vllm/lora/model_manager.py b/vllm/lora/model_manager.py index 8b5d97c1e2..2c4be56786 100644 --- a/vllm/lora/model_manager.py +++ b/vllm/lora/model_manager.py @@ -132,8 +132,13 @@ class LoRAModelManager: and self.model.is_3d_moe_weight and not self._enable_mixed_moe_lora_format ) + # Shared MoE adapters: w13 lora_A / w2 lora_B shared across experts, + # stored as pre-stacked experts.w{1,2,3} tensors (startup opt-in). + self._enable_moe_shared_loras = is_moe and lora_config.enable_moe_shared_loras self.packed_modules_mapping = process_packed_modules_mapping( - self.model, force_2d_moe=self._enable_mixed_moe_lora_format + self.model, + force_2d_moe=self._enable_mixed_moe_lora_format, + enable_moe_shared_loras=self._enable_moe_shared_loras, ) self._is_non_gated_moe = is_moe and self.model.is_non_gated_moe self._use_ep = bool( @@ -761,6 +766,16 @@ class LoRAModelManager: if lora_model.check_lora_name(replaced_module_name): module_name = replaced_module_name if module_name.endswith(".experts"): + if self._enable_moe_shared_loras: + lora_model.loras[module_name] = ( + PackedLoRALayerWeights.pack_moe_stacked( + replacement_loras, + module_name, + ) + ) + for module in packed_module_names: + lora_model.loras.pop(module, None) + continue if self._is_non_gated_moe and len(replacement_loras) > 0: replacement_loras = self._pad_lora_pairs_to_triplets( replacement_loras @@ -1019,20 +1034,10 @@ class LoRAModelManager: module: FusedMoEWithLoRA, module_name: str, ) -> None: - """Slice the cached LoRA tensors down to this rank's local experts. + """Slice cached LoRA tensors down to this rank's local experts. - The 2D MoE checkpoint enters as a list of per-(w1/w2/w3) tensors of - shape (num_experts, rank, in) / (num_experts, out, rank). When EP - is active each rank only owns local_num_experts; without this slice - the CPU LoRAModel keeps the full global weight and set_lora has to - re-slice on every activation. - - With the load-time / pack-time slicing in - ``_restrict_to_local_experts``, the stacked tensors already match - ``local_num_experts`` and the inner branch becomes a no-op. The - guard remains so checkpoints that bypassed the pre-slicing (e.g. - ``.bin``/``.pt`` adapters with weights mappers we don't recognize) - still get sliced here. + Shared factors have expert dimension one and remain available on every + rank. Per-expert factors are narrowed to the local expert block. """ if not module.use_ep: return @@ -1046,16 +1051,13 @@ class LoRAModelManager: expert_start = ep_rank * local_num_experts expert_end = expert_start + local_num_experts - new_lora_a: list[torch.Tensor | None] = [] - new_lora_b: list[torch.Tensor | None] = [] - for a, b in zip(module_lora.lora_a, module_lora.lora_b): - if a is not None and b is not None and a.shape[0] == global_num_experts: - a = a[expert_start:expert_end].contiguous() - b = b[expert_start:expert_end].contiguous() - new_lora_a.append(a) - new_lora_b.append(b) - module_lora.lora_a = new_lora_a - module_lora.lora_b = new_lora_b + def _slice_local(t: torch.Tensor | None) -> torch.Tensor | None: + if t is not None and t.shape[0] == global_num_experts: + return t[expert_start:expert_end].contiguous() + return t + + module_lora.lora_a = [_slice_local(a) for a in module_lora.lora_a] + module_lora.lora_b = [_slice_local(b) for b in module_lora.lora_b] def _restrict_to_local_experts( self, module_name: str, new_module_names: list[str] diff --git a/vllm/lora/utils.py b/vllm/lora/utils.py index 6b9c66b980..c5d5765bd0 100644 --- a/vllm/lora/utils.py +++ b/vllm/lora/utils.py @@ -369,7 +369,9 @@ def get_adapter_absolute_path(lora_path: str) -> str: def process_packed_modules_mapping( - model: nn.Module, force_2d_moe: bool = False + model: nn.Module, + force_2d_moe: bool = False, + enable_moe_shared_loras: bool = False, ) -> dict[str, list[str]]: if is_moe_model(model): # This method generates and returns a dictionary mapping packed module @@ -382,7 +384,17 @@ def process_packed_modules_mapping( # the engine forces the universal 2D wrapper via # enable_mixed_moe_lora_format (so 3D models can also load 2D # adapters through FusedMoEWithLoRA). - if (not model.is_3d_moe_weight) or force_2d_moe: + if enable_moe_shared_loras: + # Shared MoE adapters store one pre-stacked tensor per + # expert-projection (experts.w1/w2/w3) rather than a per-expert + # tensor list, so the packed mapping references the three stack + # names directly (drives expected_lora_modules and the packing). + packed_modules_mapping["experts"] = [ + "experts.w1", + "experts.w2", + "experts.w3", + ] + elif (not model.is_3d_moe_weight) or force_2d_moe: # Filter out malformed entries: non-gated MoE has empty # ckpt_up_proj_name which results in weight_name containing ".." # (e.g., "experts.0.." instead of "experts.0.layer_name.") diff --git a/vllm/model_executor/layers/fused_moe/experts/lora_context.py b/vllm/model_executor/layers/fused_moe/experts/lora_context.py index 117f744aee..4823291322 100644 --- a/vllm/model_executor/layers/fused_moe/experts/lora_context.py +++ b/vllm/model_executor/layers/fused_moe/experts/lora_context.py @@ -43,6 +43,11 @@ class MoELoRAContext: # try_get_optimal_moe_lora_config for Triton kernel tile configs. use_tuned_config: bool + # Shared MoE LoRA: w13 lora_A and w2 lora_B are shared across all experts + # (stored collapsed with expert-dim 1). When True, LoRAExpertsMixin expands + # them to local_num_experts via a stride-0 view before the kernel. + enable_moe_shared_loras: bool = False + # Optional dual-stream support for overlapping each (base GEMM, LoRA) # pair. When aux_stream is None, the experts.apply() path runs the # original sequential schedule. When set, base GEMM runs on the default diff --git a/vllm/model_executor/layers/fused_moe/experts/lora_experts_mixin.py b/vllm/model_executor/layers/fused_moe/experts/lora_experts_mixin.py index a47145b949..b8dc80ed18 100644 --- a/vllm/model_executor/layers/fused_moe/experts/lora_experts_mixin.py +++ b/vllm/model_executor/layers/fused_moe/experts/lora_experts_mixin.py @@ -52,10 +52,19 @@ class LoRAExpertsMixin: torch.Tensor | None, torch.Tensor | None, ]: + w13_lora_a_stacked = lora_context.w13_lora_a_stacked + if lora_context.enable_moe_shared_loras: + # w13 lora_A is shared across experts (collapsed expert-dim 1); + # broadcast to local_num_experts via a stride-0 view. The kernel + # derives num_experts from shape[1] and indexes via stride(1)==0. + w13_lora_a_stacked = tuple( + a.expand(-1, lora_context.local_num_experts, -1, -1) + for a in w13_lora_a_stacked + ) return lora_context.punica_wrapper.add_lora_w13( y, x, - lora_context.w13_lora_a_stacked, + w13_lora_a_stacked, lora_context.w13_lora_b_stacked, topk_ids, topk_weights, @@ -92,11 +101,19 @@ class LoRAExpertsMixin: top_k_num: int, add_inputs: bool = True, ) -> None: + w2_lora_b_stacked = lora_context.w2_lora_b_stacked + if lora_context.enable_moe_shared_loras: + # w2 lora_B is shared across experts (collapsed expert-dim 1); + # broadcast to local_num_experts via a stride-0 view. + w2_lora_b_stacked = tuple( + b.expand(-1, lora_context.local_num_experts, -1, -1) + for b in w2_lora_b_stacked + ) lora_context.punica_wrapper.add_lora_w2( y, x, lora_context.w2_lora_a_stacked, - lora_context.w2_lora_b_stacked, + w2_lora_b_stacked, topk_weights, sorted_token_ids_lora, expert_ids_lora, diff --git a/vllm/models/inkling/nvidia/logits_processor.py b/vllm/models/inkling/nvidia/logits_processor.py index b1b82659ff..2e142c5961 100644 --- a/vllm/models/inkling/nvidia/logits_processor.py +++ b/vllm/models/inkling/nvidia/logits_processor.py @@ -1,6 +1,22 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Inkling logits processor with muP output scaling.""" +"""Inkling logits processor (muP + LoRA aware). + +Inkling divides the final logits by a muP width multiplier +(``logits_mup_width_multiplier``). This applies it two ways, depending on +whether an lm_head LoRA is attached: + +* No LoRA: fold ``1/mup`` into the lm_head GEMM alpha (fp32 epilogue) -- no + separate elementwise kernel, no extra rounding, no weight mutation. +* LoRA attached: the LoRA manager wraps this layer in + ``LogitsProcessorWithLoRA``, whose ``forward`` calls + ``type(base_layer).forward(self=wrapper)`` -- so this ``forward`` runs with + ``self`` bound to the wrapper. We detect that via ``base_layer`` and take the + LoRA path: run the wrapper's ``_get_logits`` (base logits + the lm_head LoRA + delta), then divide the full logits by the multiplier so the delta is scaled + too. muP thus composes with the LoRA delta, with the dispatch as the only + model-side branching. +""" from __future__ import annotations @@ -47,6 +63,46 @@ class InklingLogitsProcessor(LogitsProcessor): lm_head: VocabParallelEmbedding, hidden_states: torch.Tensor, embedding_bias: torch.Tensor | None = None, + ) -> torch.Tensor | None: + # ``base_layer`` exists only on the LogitsProcessorWithLoRA wrapper, + # which calls this forward with ``self`` bound to the wrapper. The + # wrapper is not an ``InklingLogitsProcessor`` instance, so dispatch + # ``_lora_forward`` explicitly through the base_layer's class (it + # provides ``_get_logits``/``logits_as_input``; only ``_lora_forward`` + # lives on this class). + if hasattr(self, "base_layer"): + return type(self.base_layer)._lora_forward( + self, lm_head, hidden_states, embedding_bias + ) + return self._base_forward(lm_head, hidden_states, embedding_bias) + + def _lora_forward( + self, + lm_head: VocabParallelEmbedding, + hidden_states: torch.Tensor, + embedding_bias: torch.Tensor | None = None, + ) -> torch.Tensor | None: + # ``self`` is the LogitsProcessorWithLoRA wrapper here: ``_get_logits`` + # returns the base logits plus the lm_head LoRA delta. Apply the muP + # divisor on the full logits so the LoRA delta is scaled too. + mup_multiplier = self.base_layer.logits_mup_width_multiplier + mup = 1.0 / mup_multiplier if mup_multiplier else None + if self.logits_as_input: + logits = hidden_states + else: + logits = self._get_logits(hidden_states, lm_head, embedding_bias) + # TODO: fuse this multiplication + if logits is not None and mup: + assert self.base_layer.soft_cap is None + assert self.base_layer.scale == 1.0 + logits = logits * mup + return logits + + def _base_forward( + self, + lm_head: VocabParallelEmbedding, + hidden_states: torch.Tensor, + embedding_bias: torch.Tensor | None = None, ) -> torch.Tensor | None: mup = self.logits_mup_width_multiplier if not mup: diff --git a/vllm/models/inkling/nvidia/model.py b/vllm/models/inkling/nvidia/model.py index 39f8ea5375..bf7f700dee 100644 --- a/vllm/models/inkling/nvidia/model.py +++ b/vllm/models/inkling/nvidia/model.py @@ -24,6 +24,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead from vllm.model_executor.models.interfaces import ( MultiModalEmbeddings, + SupportsLoRA, SupportsMultiModal, SupportsPP, ) @@ -360,7 +361,7 @@ class InklingModel(nn.Module): return self.norm(hidden_states) -class _TmlForCausalLMBase(nn.Module, SupportsPP): +class _TmlForCausalLMBase(nn.Module, SupportsPP, SupportsLoRA): """Shared text-backbone causal-LM scaffolding for both entry classes.""" hf_to_vllm_mapper = WeightsMapper( @@ -380,6 +381,8 @@ class _TmlForCausalLMBase(nn.Module, SupportsPP): "model.llm.embed": "model.embed_tokens", "model.llm.norm": "model.norm", "model.llm.unembed": "lm_head", + "language_model.layers.": "model.layers.", + "language_model.lm_head.": "lm_head.", }, orig_to_new_suffix={ # NVFP4 scale @@ -389,6 +392,13 @@ class _TmlForCausalLMBase(nn.Module, SupportsPP): ".w2_weight.scale2": ".w2_weight_scale_2", }, ) + packed_modules_mapping = { + "qkvr": ["wq_du", "wk_dv", "wv_dv", "wr_du"], + "w13": ["w1", "w3"], + } + embedding_modules = { + "lm_head": "output_embeddings", + } def _build( self, diff --git a/vllm/models/inkling/nvidia/moe.py b/vllm/models/inkling/nvidia/moe.py index 3bb281c963..255c9328c8 100644 --- a/vllm/models/inkling/nvidia/moe.py +++ b/vllm/models/inkling/nvidia/moe.py @@ -343,6 +343,71 @@ class InklingSinkExperts(nn.Module): return h @ self.w2_weight.T # (T, D) +class InklingSinkExpertsLinear(nn.Module): + """LoRA-capable implementation of the Inkling sink experts.""" + + def __init__( + self, + n_experts: int, + d_model: int, + d_mlp: int, + *, + prefix: str = "", + ) -> None: + super().__init__() + from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + RowParallelLinear, + ) + + self.n_experts = n_experts + self.d_mlp = d_mlp + total = n_experts * d_mlp + self.w13 = MergedColumnParallelLinear( + input_size=d_model, + output_sizes=[total, total], + bias=False, + prefix=f"{prefix}.w13", + ) + self.w2 = RowParallelLinear( + input_size=total, + output_size=d_model, + bias=False, + reduce_results=False, + prefix=f"{prefix}.w2", + ) + self._w2_input_pp = self.w2.input_size_per_partition + self._col_expert: torch.Tensor | None = None + + def _gamma_expand(self, gammas: torch.Tensor) -> torch.Tensor: + if self._col_expert is None or self._col_expert.device != gammas.device: + local = self._w2_input_pp + start = get_tensor_model_parallel_rank() * local + cols = torch.arange(start, start + local, device=gammas.device) + self._col_expert = (cols // self.d_mlp).long() + return gammas[:, self._col_expert] + + def load_weight(self, key: str, weight: torch.Tensor) -> list[str]: + if key == "w13_weight": + d_model = weight.shape[-1] + gate = weight[:, 0::2, :].reshape(-1, d_model).contiguous() + up = weight[:, 1::2, :].reshape(-1, d_model).contiguous() + self.w13.weight_loader(self.w13.weight, gate, 0) + self.w13.weight_loader(self.w13.weight, up, 1) + return ["w13.weight"] + w = weight.permute(1, 0, 2).reshape(weight.shape[1], -1).contiguous() + self.w2.weight_loader(self.w2.weight, w) + return ["w2.weight"] + + def forward(self, x: torch.Tensor, gammas: torch.Tensor) -> torch.Tensor: + gate_up, _ = self.w13(x) + gate, up = gate_up.chunk(2, dim=-1) + hidden_states = torch.nn.functional.silu(gate) * up + hidden_states = (hidden_states * self._gamma_expand(gammas)).to(x.dtype) + output, _ = self.w2(hidden_states) + return output + + class InklingMoE(nn.Module): def __init__( self, @@ -417,7 +482,12 @@ class InklingMoE(nn.Module): layer_id ), f"layer {layer_id}: NVFP4 shared experts are not supported" - self.sink_experts = InklingSinkExperts( + sink_experts_cls = ( + InklingSinkExpertsLinear + if get_current_vllm_config().lora_config is not None + else InklingSinkExperts + ) + self.sink_experts = sink_experts_cls( n_experts=n_shared, d_model=config.hidden_size, d_mlp=config.intermediate_size,