[LoRA] Support 2D and 3D MoE LoRA adapter at the same time (#42242)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Jee Jee Li
2026-05-18 15:22:26 +08:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Roger Wang
parent e3aeee5ff8
commit 7d5b033782
16 changed files with 391 additions and 9 deletions
+51
View File
@@ -248,6 +248,57 @@ Now, you can specify a base_model_name alongside the name and path using JSON fo
To provide the backward compatibility support, you can still use the old key-value format (name=path), but the `base_model_name` will remain unspecified in that case.
## Mixing 2D and 3D MoE LoRA Adapters
To serve 2D-format(based on `megatron`) and 3D-format (based on `peft`) adapters from the same engine instance, start the server with `--enable-mixed-moe-lora-format`
and declare the layout of each adapter explicitly via the `is_3d_lora_weight` field.
Server startup (static modules):
```bash
vllm serve Qwen/Qwen3.6-35B-A3B \
--enable-lora \
--enable-mixed-moe-lora-format \
--tensor-parallel-size 4 \
--enable-expert-parallel \
--lora-modules \
'{"name": "lora-2d", "path": "jeeejeee/qwen36-35ba3b-2d-weights-poken-lora", "is_3d_lora_weight": false}' \
'{"name": "lora-3d", "path": "jeeejeee/qwen36-35ba3b-moe-all-linear-poken-lora", "is_3d_lora_weight": true}'
```
Dynamic load via `/v1/load_lora_adapter`:
```bash
curl -X POST http://localhost:8000/v1/load_lora_adapter \
-H "Content-Type: application/json" \
-d '{
"lora_name": "lora-3d",
"lora_path": "/path/to/3d-format-lora",
"is_3d_lora_weight": true
}'
```
!!! warning "You must know your adapter's layout"
Under `--enable-mixed-moe-lora-format`, vLLM trusts whatever
`is_3d_lora_weight` the caller declares — it does **not** inspect the
checkpoint to verify. A wrong declaration will load weights into the
wrong stacked buffers and silently produce garbage outputs, with no
error at load time. Confirm the layout before serving:
- **2D (per-expert, megatron-style)** → set `is_3d_lora_weight: false`.
Adapter keys look like `...experts.{idx}.gate_proj.lora_A.weight`,
`...experts.{idx}.up_proj.lora_A.weight`,
`...experts.{idx}.down_proj.lora_A.weight` — one set per expert.
- **3D (fused, peft-style)** → set `is_3d_lora_weight: true`.
Adapter keys look like `...experts.gate_up_proj.lora_A.weight`,
`...experts.down_proj.lora_A.weight` — a single tensor that stacks
all experts on the leading dim.
When `--enable-mixed-moe-lora-format` is **not** set, `is_3d_lora_weight`
is ignored: vLLM picks the wrapper from the base model's
`is_3d_moe_weight` and the adapter is required to match. The field is
also ignored for non-MoE models.
## LoRA model lineage in model card
The new format of `--lora-modules` is mainly to support the display of parent model information in the model card. Here's an explanation of how your current response supports this:
+10
View File
@@ -313,6 +313,16 @@ def qwen35_vl_lora_files():
return snapshot_download(repo_id="jeeejeee/qwen35-4b-all-linear-pokemon-lora")
@pytest.fixture(scope="session")
def qwen36_moe_2d_lora_files():
return snapshot_download(repo_id="jeeejeee/qwen36-35ba3b-2d-weights-poken-lora")
@pytest.fixture(scope="session")
def qwen36_moe_3d_lora_files():
return snapshot_download(repo_id="jeeejeee/qwen36-35ba3b-moe-all-linear-poken-lora")
@pytest.fixture
def reset_default_device():
"""
+156
View File
@@ -0,0 +1,156 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import vllm
import vllm.config
from vllm.assets.image import ImageAsset
from vllm.lora.request import LoRARequest
from ..utils import multi_gpu_test
MODEL_PATH = "Qwen/Qwen3.6-35B-A3B"
LORA_2D_ID = 1
LORA_3D_ID = 2
PROMPT_TEMPLATE = """<|im_start|>user
<|vision_start|><|image_pad|><|vision_end|>What is in the image?<|im_end|>
<|im_start|>assistant
<think>
</think>
"""
# Visual captioning prompts: each image will be paired with one LoRA in the
# mixed-batch case so we can check per-prompt routing.
VL_TEST_IMAGES = [
ImageAsset("stop_sign"),
ImageAsset("cherry_blossom"),
]
def _build_prompts() -> list[dict]:
return [
{
"prompt": PROMPT_TEMPLATE,
"multi_modal_data": {"image": asset.pil_image},
}
for asset in VL_TEST_IMAGES
]
def _generate(llm: vllm.LLM, lora_request) -> list[str]:
outputs = llm.generate(
_build_prompts(),
vllm.SamplingParams(temperature=0, max_tokens=128),
lora_request=lora_request,
)
return [out.outputs[0].text.strip() for out in outputs]
def _run_mixed_2d_3d_lora_test(
lora_2d_files: str,
lora_3d_files: str,
tensor_parallel_size: int,
fully_sharded_loras: bool,
) -> None:
llm = vllm.LLM(
model=MODEL_PATH,
max_model_len=4096,
enable_lora=True,
enable_mixed_moe_lora_format=True,
max_loras=2,
max_lora_rank=8,
max_num_seqs=4,
enforce_eager=True,
tensor_parallel_size=tensor_parallel_size,
enable_expert_parallel=not fully_sharded_loras,
fully_sharded_loras=fully_sharded_loras,
trust_remote_code=True,
enable_tower_connector_lora=True,
mm_processor_cache_gb=0,
limit_mm_per_prompt={"image": 1},
compilation_config=vllm.config.CompilationConfig(
cudagraph_specialize_lora=False,
),
)
lora_2d = LoRARequest(
"lora_2d",
LORA_2D_ID,
lora_2d_files,
is_3d_lora_weight=False,
)
lora_3d = LoRARequest(
"lora_3d",
LORA_3D_ID,
lora_3d_files,
is_3d_lora_weight=True,
)
# Reference: each adapter alone over both prompts.
outputs_2d_alone = _generate(llm, lora_2d)
outputs_3d_alone = _generate(llm, lora_3d)
assert len(outputs_2d_alone) == len(VL_TEST_IMAGES)
assert len(outputs_3d_alone) == len(VL_TEST_IMAGES)
for text in outputs_2d_alone + outputs_3d_alone:
assert text, "Empty output from single-adapter LoRA generation"
# Mixed batch: prompt 0 uses the 2D adapter, prompt 1 uses the 3D
# adapter. Per-prompt outputs must match the standalone runs.
mixed_outputs = _generate(llm, [lora_2d, lora_3d])
assert mixed_outputs[0] == outputs_2d_alone[0], (
f"Mixed-batch 2D output {mixed_outputs[0]!r} does not match "
f"standalone 2D output {outputs_2d_alone[0]!r}"
)
assert mixed_outputs[1] == outputs_3d_alone[1], (
f"Mixed-batch 3D output {mixed_outputs[1]!r} does not match "
f"standalone 3D output {outputs_3d_alone[1]!r}"
)
# Reverse assignment: neither adapter should be silently aliased.
swapped_outputs = _generate(llm, [lora_3d, lora_2d])
assert swapped_outputs[0] == outputs_3d_alone[0], (
f"Swapped-batch 3D output {swapped_outputs[0]!r} does not match "
f"standalone 3D output {outputs_3d_alone[0]!r}"
)
assert swapped_outputs[1] == outputs_2d_alone[1], (
f"Swapped-batch 2D output {swapped_outputs[1]!r} does not match "
f"standalone 2D output {outputs_2d_alone[1]!r}"
)
@pytest.mark.skip(reason="This model is too big, so skip this test temporarily.")
@pytest.mark.parametrize("fully_sharded_loras", [False, True])
@multi_gpu_test(num_gpus=2)
def test_qwen36_moe_mixed_2d_3d_lora_tp2(
qwen36_moe_2d_lora_files,
qwen36_moe_3d_lora_files,
fully_sharded_loras,
):
_run_mixed_2d_3d_lora_test(
lora_2d_files=qwen36_moe_2d_lora_files,
lora_3d_files=qwen36_moe_3d_lora_files,
tensor_parallel_size=2,
fully_sharded_loras=fully_sharded_loras,
)
@pytest.mark.skip(reason="This model is too big, so skip this test temporarily.")
@pytest.mark.parametrize("fully_sharded_loras", [False, True])
@multi_gpu_test(num_gpus=4)
def test_qwen36_moe_mixed_2d_3d_lora_tp4(
qwen36_moe_2d_lora_files,
qwen36_moe_3d_lora_files,
fully_sharded_loras,
):
_run_mixed_2d_3d_lora_test(
lora_2d_files=qwen36_moe_2d_lora_files,
lora_3d_files=qwen36_moe_3d_lora_files,
tensor_parallel_size=4,
fully_sharded_loras=fully_sharded_loras,
)
+7 -2
View File
@@ -77,6 +77,7 @@ def test_qwen3moe_lora(qwen3moe_lora_files):
enforce_eager=True,
trust_remote_code=True,
enable_chunked_prefill=True,
enable_mixed_moe_lora_format=True,
)
generate_and_test(llm, qwen3moe_lora_files, lora_id=1)
@@ -85,7 +86,8 @@ def test_qwen3moe_lora(qwen3moe_lora_files):
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("ep", [False, True])
def test_qwen3moe_lora_tp2(ep, qwen3moe_lora_files):
@pytest.mark.parametrize("enable_mixed_moe_lora_format", [False, True])
def test_qwen3moe_lora_tp2(ep, qwen3moe_lora_files, enable_mixed_moe_lora_format):
llm = vllm.LLM(
MODEL_PATH,
max_model_len=1024,
@@ -94,6 +96,7 @@ def test_qwen3moe_lora_tp2(ep, qwen3moe_lora_files):
trust_remote_code=True,
enable_expert_parallel=ep,
tensor_parallel_size=2,
enable_mixed_moe_lora_format=enable_mixed_moe_lora_format,
)
generate_and_test(llm, qwen3moe_lora_files, lora_id=1)
@@ -102,7 +105,8 @@ def test_qwen3moe_lora_tp2(ep, qwen3moe_lora_files):
@multi_gpu_test(num_gpus=4)
@pytest.mark.parametrize("ep", [False, True])
def test_qwen3moe_lora_tp4(ep, qwen3moe_lora_files):
@pytest.mark.parametrize("enable_mixed_moe_lora_format", [False, True])
def test_qwen3moe_lora_tp4(ep, qwen3moe_lora_files, enable_mixed_moe_lora_format):
llm = vllm.LLM(
MODEL_PATH,
max_model_len=1024,
@@ -111,6 +115,7 @@ def test_qwen3moe_lora_tp4(ep, qwen3moe_lora_files):
trust_remote_code=True,
enable_expert_parallel=ep,
tensor_parallel_size=4,
enable_mixed_moe_lora_format=enable_mixed_moe_lora_format,
)
generate_and_test(llm, qwen3moe_lora_files, lora_id=1)
+7
View File
@@ -71,6 +71,12 @@ class LoRAConfig:
for variable LoRA usage patterns at the cost of increased startup time and
memory usage. Only takes effect when cudagraph_specialize_lora is True.
"""
enable_mixed_moe_lora_format: bool = False
"""If True, force the engine to use the universal 2D MoE LoRA wrapper
(`FusedMoEWithLoRA`) regardless of the model's `is_3d_moe_weight` flag, so
that 2D-format and 3D-format MoE LoRA adapters can be served in the same
deployment. Only meaningful forMoE models; ignored otherwise. Default False
keeps the existing model-driven behavior."""
def compute_hash(self) -> str:
"""
@@ -90,6 +96,7 @@ class LoRAConfig:
factors.append(self.fully_sharded_loras)
factors.append(self.lora_dtype)
factors.append(self.enable_tower_connector_lora)
factors.append(self.enable_mixed_moe_lora_format)
# target_modules affects which modules get LoRA applied
factors.append(
tuple(sorted(self.target_modules)) if self.target_modules else None
+6
View File
@@ -584,6 +584,7 @@ class EngineArgs:
lora_target_modules: list[str] | None = LoRAConfig.target_modules
enable_tower_connector_lora: bool = LoRAConfig.enable_tower_connector_lora
specialize_active_lora: bool = LoRAConfig.specialize_active_lora
enable_mixed_moe_lora_format: bool = LoRAConfig.enable_mixed_moe_lora_format
ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight
num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override
@@ -1272,6 +1273,10 @@ class EngineArgs:
lora_group.add_argument(
"--specialize-active-lora", **lora_kwargs["specialize_active_lora"]
)
lora_group.add_argument(
"--enable-mixed-moe-lora-format",
**lora_kwargs["enable_mixed_moe_lora_format"],
)
# Observability arguments
observability_kwargs = get_kwargs(ObservabilityConfig)
@@ -2009,6 +2014,7 @@ class EngineArgs:
target_modules=self.lora_target_modules,
enable_tower_connector_lora=self.enable_tower_connector_lora,
specialize_active_lora=self.specialize_active_lora,
enable_mixed_moe_lora_format=self.enable_mixed_moe_lora_format,
max_cpu_loras=self.max_cpu_loras
if self.max_cpu_loras and self.max_cpu_loras > 0
else None,
@@ -16,3 +16,4 @@ class LoRAModulePath:
name: str
path: str
base_model_name: str | None = None
is_3d_lora_weight: bool = False
+4 -1
View File
@@ -121,7 +121,9 @@ class OpenAIServingModels:
return
for lora in self.static_lora_modules:
load_request = LoadLoRAAdapterRequest(
lora_path=lora.path, lora_name=lora.name
lora_path=lora.path,
lora_name=lora.name,
is_3d_lora_weight=lora.is_3d_lora_weight,
)
load_result = await self.load_lora_adapter(
request=load_request, base_model_name=lora.base_model_name
@@ -177,6 +179,7 @@ class OpenAIServingModels:
lora_int_id=lora_int_id,
lora_path=lora_path,
load_inplace=request.load_inplace,
is_3d_lora_weight=request.is_3d_lora_weight,
)
if base_model_name is not None and self.is_base_model(base_model_name):
lora_request.base_model_name = base_model_name
@@ -37,6 +37,7 @@ def attach_router(app: FastAPI):
"lora_name": "body.name",
"lora_path": "body.src",
"load_inplace": "body.load_inplace || `false`",
"is_3d_lora_weight": "body.is_3d_lora_weight || `false`",
},
)
@router.post("/v1/load_lora_adapter", dependencies=[Depends(validate_json_request)])
+1
View File
@@ -8,6 +8,7 @@ class LoadLoRAAdapterRequest(BaseModel):
lora_name: str
lora_path: str
load_inplace: bool = False
is_3d_lora_weight: bool = False
class UnloadLoRAAdapterRequest(BaseModel):
+4
View File
@@ -37,6 +37,10 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
# For non-gated MoE (is_act_and_mul=False), only 1 slice is needed
# since there's only up_proj (w1), not gate_proj + up_proj (w1 + w3)
self._w13_slices = 2 if base_layer.moe_config.is_act_and_mul else 1
# Mirrors per-(lora_id) layout of `self.lora_a_stacked` (built in
# `create_lora_weights`) so `create_dummy_lora`'s n_slices fallback
# matches `lora_a_stacked` length under EP.
self.n_slices = base_layer.local_num_experts * (self._w13_slices + 1)
self.base_layer.ensure_moe_quant_config_init()
if getattr(self.base_layer.quant_method, "supports_internal_mk", False):
+7
View File
@@ -29,12 +29,17 @@ class LoRAModel:
lora_model_id: int,
rank: int,
loras: dict[str, LoRALayerWeights],
is_3d_lora_weight: bool = False,
) -> None:
"""
Args:
lora_model_id: The integer id for the lora model.
rank: lora rank.
loras: module name -> weights for lora-replaced layers.
is_3d_lora_weight: Whether the on-disk MoE adapter is in the 3D
fused (gate_up_proj / down_proj) layout. Propagated from the
originating LoRARequest. Only consulted by the LoRA model
manager when enable_mixed_moe_lora_format is on.
"""
self.id = lora_model_id
@@ -44,6 +49,7 @@ class LoRAModel:
)
self.rank = rank
self.loras: dict[str, LoRALayerWeights] = loras
self.is_3d_lora_weight = is_3d_lora_weight
def clone(self, lora_model_id: int) -> "LoRAModel":
"""Return a copy of the object with different ids.
@@ -53,6 +59,7 @@ class LoRAModel:
lora_model_id,
rank=self.rank,
loras=self.loras.copy(),
is_3d_lora_weight=self.is_3d_lora_weight,
)
def get_lora(self, module_name: str) -> LoRALayerWeights | None:
+117 -3
View File
@@ -102,7 +102,6 @@ class LoRAModelManager:
self.max_num_batched_tokens = math.ceil(max_num_batched_tokens / 8) * 8
self.lora_index_to_id: list[int | None] = [None] * self.lora_slots
self.vocab_size = vocab_size
self.packed_modules_mapping = process_packed_modules_mapping(self.model)
self.is_pooling_model = is_pooling_model(self.model)
self.packed_modules: dict[str, list[str]] = {}
@@ -110,7 +109,20 @@ class LoRAModelManager:
# Dict instead of a set for compatibility with LRUCache.
self._last_mapping: LoRAMapping | None = None
is_moe = is_moe_model(self.model)
self._is_3d_moe_model = is_moe and self.model.is_3d_moe_weight
# Whether the underlying model class declares 3D fused MoE weights.
self._model_is_3d_moe = is_moe and self.model.is_3d_moe_weight
# When the engine is started with enable_mixed_moe_lora_format=True
# we force the universal 2D wrapper (FusedMoEWithLoRA) regardless of
# the model's 3D flag, so 2D and 3D adapters can coexist.
self._enable_mixed_moe_lora_format = (
is_moe and lora_config.enable_mixed_moe_lora_format
)
self._is_3d_moe_model = (
self._model_is_3d_moe and not self._enable_mixed_moe_lora_format
)
self.packed_modules_mapping = process_packed_modules_mapping(
self.model, force_2d_moe=self._enable_mixed_moe_lora_format
)
self._is_non_gated_moe = is_moe and self.model.is_non_gated_moe
self._init_punica_wrapper(max_num_batched_tokens, vllm_config)
self._create_lora_modules()
@@ -748,7 +760,16 @@ class LoRAModelManager:
if isinstance(module, FusedMoE3DWithLoRA):
self._stack_moe_lora_weights(lora_model, module, module_name)
elif isinstance(module, FusedMoEWithLoRA):
self._slice_moe_lora_ep(lora_model, module, module_name)
# When mixed mode is enabled the universal 2D wrapper has to
# absorb both 2D and 3D-format adapters. 3D-format adapters
# need to be split into per-(w1, w2, w3) tensors before the
# 2D set_lora can copy them into the stacked buffers.
if self._enable_mixed_moe_lora_format and getattr(
lora_model, "is_3d_lora_weight", False
):
self._convert_3d_to_2d_moe_lora(lora_model, module, module_name)
else:
self._slice_moe_lora_ep(lora_model, module, module_name)
first_lora: LoRALayerWeights = next(iter(lora_model.loras.values()))
assert first_lora.lora_a is not None
@@ -871,6 +892,99 @@ class LoRAModelManager:
module_lora.lora_a = lora_a
module_lora.lora_b = lora_b
def _convert_3d_to_2d_moe_lora(
self,
lora_model: LoRAModel,
module: FusedMoEWithLoRA,
module_name: str,
) -> None:
"""Convert a 3D-format MoE LoRA checkpoint into the 2D pack layout
that `FusedMoEWithLoRA.set_lora` expects.
On disk the 3D PEFT layout stores two flat tensor pairs per layer:
- `{module}.base_layer.lora_{A,B}`: gate_up_proj, with shapes
`(rank * num_experts, hidden)` / `(intermediate * 2,
rank * num_experts)`
- `{module}.lora_{A,B}`: down_proj, with shapes
`(rank * num_experts, intermediate)` / `(hidden,
rank * num_experts)`
The 2D wrapper expects three stacked per-expert tensors,
`[w1, w2, w3]`, with `(num_experts, rank, in)` for lora_a and
`(num_experts, out, rank)` for lora_b. In the 3D layout w1
(gate_proj) and w3 (up_proj) share the rank-r intermediate
representation, so both halves use the same lora_a tensor.
Only invoked when `enable_mixed_moe_lora_format=True` and the
source LoRARequest declares `is_3d_lora_weight=True`.
"""
gate_up_proj_lora = self._get_lora_layer_weights(
lora_model, module_name + ".base_layer"
)
down_proj_lora = self._get_lora_layer_weights(lora_model, module_name)
if gate_up_proj_lora is None or down_proj_lora is None:
# Either the adapter omits the experts entirely or the file
# layout differs from what this path supports; leave the entry
# untouched so set_lora can raise a clear error if needed.
return
local_num_experts = module.base_layer.local_num_experts
global_num_experts = module.base_layer.global_num_experts
ep_rank = module.base_layer.ep_rank
expert_start = ep_rank * local_num_experts
expert_end = expert_start + local_num_experts
# Reshape and EP-slice into per-expert 3D tensors. This mirrors
# `_stack_moe_lora_weights`; for non-EP runs the slice is a no-op.
gate_up_a = gate_up_proj_lora.lora_a.reshape(
global_num_experts, -1, gate_up_proj_lora.lora_a.shape[-1]
)[expert_start:expert_end].contiguous()
gate_up_b = (
gate_up_proj_lora.lora_b.reshape(
gate_up_proj_lora.lora_b.shape[0], -1, global_num_experts
)[..., expert_start:expert_end]
.permute(2, 0, 1)
.contiguous()
)
down_a = down_proj_lora.lora_a.reshape(
global_num_experts, -1, down_proj_lora.lora_a.shape[-1]
)[expert_start:expert_end].contiguous()
down_b = (
down_proj_lora.lora_b.reshape(
down_proj_lora.lora_b.shape[0], -1, global_num_experts
)[..., expert_start:expert_end]
.permute(2, 0, 1)
.contiguous()
)
# Split the fused gate_up_proj output dim into separate w1 / w3
# halves. GPT-OSS interleaves them along the output dim, all other
# 3D MoE checkpoints we know about concatenate them.
intermediate_x2 = gate_up_b.shape[1]
if intermediate_x2 % 2 != 0:
raise ValueError(
"Expected gate_up_proj LoRA-B output dim to be 2 * intermediate, "
f"got {intermediate_x2}."
)
intermediate = intermediate_x2 // 2
base_arch = self.model.config.architectures[0]
if base_arch == "GptOssForCausalLM":
w1_b = gate_up_b[:, ::2, :].contiguous()
w3_b = gate_up_b[:, 1::2, :].contiguous()
else:
w1_b = gate_up_b[:, :intermediate, :].contiguous()
w3_b = gate_up_b[:, intermediate:, :].contiguous()
# In the 3D layout w1 and w3 share the same rank-r mid
# representation, so they reuse the same lora_a tensor. The 2D
# wrapper's set_lora copies whatever it gets here into independent
# per-slice buffers, so the sharing is purely a CPU-side memory
# optimization and does not affect numerics.
down_proj_lora.lora_a = [gate_up_a, down_a, gate_up_a]
down_proj_lora.lora_b = [w1_b, down_b, w3_b]
# Drop the redundant base_layer entry to avoid double pin_memory
# and to keep the activation path looking up only the wrapper key.
lora_model.loras.pop(module_name + ".base_layer", None)
def _slice_moe_lora_ep(
self,
lora_model: LoRAModel,
+7
View File
@@ -28,6 +28,13 @@ class LoRARequest(
base_model_name: str | None = msgspec.field(default=None)
tensorizer_config_dict: dict | None = None
load_inplace: bool = False
is_3d_lora_weight: bool = False
"""Whether this adapter's MoE weights are stored in the 3D fused
`gate_up_proj` / `down_proj` layout (one fused tensor per layer) or the
2D per-expert split layout (separate `gate_proj` / `up_proj` / `down_proj`
tensors per expert). Only consulted when the engine is started with
`enable_mixed_moe_lora_format=True`; otherwise it is ignored and the
on-disk format is inferred from the base model."""
def __post_init__(self):
if self.lora_int_id < 1:
+8 -3
View File
@@ -355,7 +355,9 @@ def get_adapter_absolute_path(lora_path: str) -> str:
return local_snapshot_path
def process_packed_modules_mapping(model: nn.Module) -> dict[str, list[str]]:
def process_packed_modules_mapping(
model: nn.Module, force_2d_moe: bool = False
) -> dict[str, list[str]]:
if is_moe_model(model):
if moe_packed_mapping := get_moe_expert_mapping(model):
# This method generates and returns a dictionary mapping packed module
@@ -364,8 +366,11 @@ def process_packed_modules_mapping(model: nn.Module) -> dict[str, list[str]]:
# the expert indices are expanded based on the configured number
# of routed experts.
packed_modules_mapping = get_packed_modules_mapping(model)
if not model.is_3d_moe_weight:
# 3D MoE LoRA does not need `packed_modules_mapping`
# The 2D mapping is needed when the model itself is 2D, or when
# the engine forces the universal 2D wrapper via
# enable_mixed_moe_lora_format (so 3D models can also load 2D
# adapters through FusedMoEWithLoRA).
if (not model.is_3d_moe_weight) or force_2d_moe:
# Filter out malformed entries: non-gated MoE has empty
# ckpt_up_proj_name which results in weight_name containing ".."
# (e.g., "experts.0.." instead of "experts.0.layer_name.")
+4
View File
@@ -141,6 +141,10 @@ class WorkerLoRAManager:
weights_mapper=hf_to_vllm_mapper,
skip_prefixes=lora_skip_prefixes,
)
# Stamp the on-disk MoE layout onto the loaded model so the
# adapter manager can route 3D-format checkpoints through the
# 3D->2D conversion when running under the universal 2D wrapper.
lora.is_3d_lora_weight = lora_request.is_3d_lora_weight
except FileNotFoundError as e:
# FileNotFoundError should be raised if both