[Model] Add Inkling LoRA support [4/N] (#48884)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Bugen Zhao
Giancarlo Delfin
Isotr0py
Isotr0py
Jee Jee Li
Roger Wang
Yifan Qiao
Claude Fable 5
OpenAI Codex
parent
fe784ff22e
commit
f3e9497e92
@@ -6,7 +6,9 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.lora.utils import get_supported_lora_modules
|
||||
from vllm.models.inkling.nvidia import moe
|
||||
from vllm.models.inkling.nvidia.model import _TmlForCausalLMBase
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -62,6 +64,29 @@ def test_gate_uses_ll_bf16_gemm_through_token_limit(
|
||||
assert logits.dtype == torch.float32
|
||||
|
||||
|
||||
def test_gate_is_not_a_lora_target() -> None:
|
||||
model = torch.nn.Module()
|
||||
model.gate = moe.InklingGate(
|
||||
d_model=4,
|
||||
n_routed_experts=5,
|
||||
n_shared_experts=2,
|
||||
experts_per_token=2,
|
||||
route_scale=1.0,
|
||||
)
|
||||
|
||||
assert "gate" not in get_supported_lora_modules(model)
|
||||
|
||||
|
||||
def test_custom_embedding_is_not_a_lora_target() -> None:
|
||||
model = torch.nn.Module()
|
||||
model.embedding_modules = _TmlForCausalLMBase.embedding_modules
|
||||
|
||||
supported = get_supported_lora_modules(model)
|
||||
|
||||
assert "embed_tokens" not in supported
|
||||
assert "lm_head" in supported
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)])
|
||||
def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
|
||||
experts = SimpleNamespace(
|
||||
|
||||
Reference in New Issue
Block a user