[Model] Add Inkling LoRA support [4/N] (#48884)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Woosuk Kwon
2026-07-17 09:52:15 +08:00
committed by GitHub
co-authored by Bugen Zhao Giancarlo Delfin Isotr0py Isotr0py Jee Jee Li Roger Wang Yifan Qiao Claude Fable 5 OpenAI Codex
parent fe784ff22e
commit f3e9497e92
12 changed files with 337 additions and 47 deletions
@@ -6,7 +6,9 @@ from types import SimpleNamespace
import pytest
import torch
from vllm.lora.utils import get_supported_lora_modules
from vllm.models.inkling.nvidia import moe
from vllm.models.inkling.nvidia.model import _TmlForCausalLMBase
from vllm.platforms import current_platform
@@ -62,6 +64,29 @@ def test_gate_uses_ll_bf16_gemm_through_token_limit(
assert logits.dtype == torch.float32
def test_gate_is_not_a_lora_target() -> None:
model = torch.nn.Module()
model.gate = moe.InklingGate(
d_model=4,
n_routed_experts=5,
n_shared_experts=2,
experts_per_token=2,
route_scale=1.0,
)
assert "gate" not in get_supported_lora_modules(model)
def test_custom_embedding_is_not_a_lora_target() -> None:
model = torch.nn.Module()
model.embedding_modules = _TmlForCausalLMBase.embedding_modules
supported = get_supported_lora_modules(model)
assert "embed_tokens" not in supported
assert "lm_head" in supported
@pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)])
def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
experts = SimpleNamespace(