[LoRA] Support 2D and 3D MoE LoRA adapter at the same time (#42242)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Roger Wang
parent
e3aeee5ff8
commit
7d5b033782
@@ -248,6 +248,57 @@ Now, you can specify a base_model_name alongside the name and path using JSON fo
|
||||
|
||||
To provide the backward compatibility support, you can still use the old key-value format (name=path), but the `base_model_name` will remain unspecified in that case.
|
||||
|
||||
## Mixing 2D and 3D MoE LoRA Adapters
|
||||
|
||||
To serve 2D-format(based on `megatron`) and 3D-format (based on `peft`) adapters from the same engine instance, start the server with `--enable-mixed-moe-lora-format`
|
||||
and declare the layout of each adapter explicitly via the `is_3d_lora_weight` field.
|
||||
|
||||
Server startup (static modules):
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3.6-35B-A3B \
|
||||
--enable-lora \
|
||||
--enable-mixed-moe-lora-format \
|
||||
--tensor-parallel-size 4 \
|
||||
--enable-expert-parallel \
|
||||
--lora-modules \
|
||||
'{"name": "lora-2d", "path": "jeeejeee/qwen36-35ba3b-2d-weights-poken-lora", "is_3d_lora_weight": false}' \
|
||||
'{"name": "lora-3d", "path": "jeeejeee/qwen36-35ba3b-moe-all-linear-poken-lora", "is_3d_lora_weight": true}'
|
||||
```
|
||||
|
||||
Dynamic load via `/v1/load_lora_adapter`:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8000/v1/load_lora_adapter \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"lora_name": "lora-3d",
|
||||
"lora_path": "/path/to/3d-format-lora",
|
||||
"is_3d_lora_weight": true
|
||||
}'
|
||||
```
|
||||
|
||||
!!! warning "You must know your adapter's layout"
|
||||
Under `--enable-mixed-moe-lora-format`, vLLM trusts whatever
|
||||
`is_3d_lora_weight` the caller declares — it does **not** inspect the
|
||||
checkpoint to verify. A wrong declaration will load weights into the
|
||||
wrong stacked buffers and silently produce garbage outputs, with no
|
||||
error at load time. Confirm the layout before serving:
|
||||
|
||||
- **2D (per-expert, megatron-style)** → set `is_3d_lora_weight: false`.
|
||||
Adapter keys look like `...experts.{idx}.gate_proj.lora_A.weight`,
|
||||
`...experts.{idx}.up_proj.lora_A.weight`,
|
||||
`...experts.{idx}.down_proj.lora_A.weight` — one set per expert.
|
||||
- **3D (fused, peft-style)** → set `is_3d_lora_weight: true`.
|
||||
Adapter keys look like `...experts.gate_up_proj.lora_A.weight`,
|
||||
`...experts.down_proj.lora_A.weight` — a single tensor that stacks
|
||||
all experts on the leading dim.
|
||||
|
||||
When `--enable-mixed-moe-lora-format` is **not** set, `is_3d_lora_weight`
|
||||
is ignored: vLLM picks the wrapper from the base model's
|
||||
`is_3d_moe_weight` and the adapter is required to match. The field is
|
||||
also ignored for non-MoE models.
|
||||
|
||||
## LoRA model lineage in model card
|
||||
|
||||
The new format of `--lora-modules` is mainly to support the display of parent model information in the model card. Here's an explanation of how your current response supports this:
|
||||
|
||||
@@ -313,6 +313,16 @@ def qwen35_vl_lora_files():
|
||||
return snapshot_download(repo_id="jeeejeee/qwen35-4b-all-linear-pokemon-lora")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen36_moe_2d_lora_files():
|
||||
return snapshot_download(repo_id="jeeejeee/qwen36-35ba3b-2d-weights-poken-lora")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def qwen36_moe_3d_lora_files():
|
||||
return snapshot_download(repo_id="jeeejeee/qwen36-35ba3b-moe-all-linear-poken-lora")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def reset_default_device():
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm
|
||||
import vllm.config
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.lora.request import LoRARequest
|
||||
|
||||
from ..utils import multi_gpu_test
|
||||
|
||||
MODEL_PATH = "Qwen/Qwen3.6-35B-A3B"
|
||||
|
||||
LORA_2D_ID = 1
|
||||
LORA_3D_ID = 2
|
||||
|
||||
PROMPT_TEMPLATE = """<|im_start|>user
|
||||
<|vision_start|><|image_pad|><|vision_end|>What is in the image?<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
"""
|
||||
|
||||
# Visual captioning prompts: each image will be paired with one LoRA in the
|
||||
# mixed-batch case so we can check per-prompt routing.
|
||||
VL_TEST_IMAGES = [
|
||||
ImageAsset("stop_sign"),
|
||||
ImageAsset("cherry_blossom"),
|
||||
]
|
||||
|
||||
|
||||
def _build_prompts() -> list[dict]:
|
||||
return [
|
||||
{
|
||||
"prompt": PROMPT_TEMPLATE,
|
||||
"multi_modal_data": {"image": asset.pil_image},
|
||||
}
|
||||
for asset in VL_TEST_IMAGES
|
||||
]
|
||||
|
||||
|
||||
def _generate(llm: vllm.LLM, lora_request) -> list[str]:
|
||||
outputs = llm.generate(
|
||||
_build_prompts(),
|
||||
vllm.SamplingParams(temperature=0, max_tokens=128),
|
||||
lora_request=lora_request,
|
||||
)
|
||||
return [out.outputs[0].text.strip() for out in outputs]
|
||||
|
||||
|
||||
def _run_mixed_2d_3d_lora_test(
|
||||
lora_2d_files: str,
|
||||
lora_3d_files: str,
|
||||
tensor_parallel_size: int,
|
||||
fully_sharded_loras: bool,
|
||||
) -> None:
|
||||
llm = vllm.LLM(
|
||||
model=MODEL_PATH,
|
||||
max_model_len=4096,
|
||||
enable_lora=True,
|
||||
enable_mixed_moe_lora_format=True,
|
||||
max_loras=2,
|
||||
max_lora_rank=8,
|
||||
max_num_seqs=4,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enable_expert_parallel=not fully_sharded_loras,
|
||||
fully_sharded_loras=fully_sharded_loras,
|
||||
trust_remote_code=True,
|
||||
enable_tower_connector_lora=True,
|
||||
mm_processor_cache_gb=0,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
compilation_config=vllm.config.CompilationConfig(
|
||||
cudagraph_specialize_lora=False,
|
||||
),
|
||||
)
|
||||
|
||||
lora_2d = LoRARequest(
|
||||
"lora_2d",
|
||||
LORA_2D_ID,
|
||||
lora_2d_files,
|
||||
is_3d_lora_weight=False,
|
||||
)
|
||||
lora_3d = LoRARequest(
|
||||
"lora_3d",
|
||||
LORA_3D_ID,
|
||||
lora_3d_files,
|
||||
is_3d_lora_weight=True,
|
||||
)
|
||||
|
||||
# Reference: each adapter alone over both prompts.
|
||||
outputs_2d_alone = _generate(llm, lora_2d)
|
||||
outputs_3d_alone = _generate(llm, lora_3d)
|
||||
|
||||
assert len(outputs_2d_alone) == len(VL_TEST_IMAGES)
|
||||
assert len(outputs_3d_alone) == len(VL_TEST_IMAGES)
|
||||
for text in outputs_2d_alone + outputs_3d_alone:
|
||||
assert text, "Empty output from single-adapter LoRA generation"
|
||||
|
||||
# Mixed batch: prompt 0 uses the 2D adapter, prompt 1 uses the 3D
|
||||
# adapter. Per-prompt outputs must match the standalone runs.
|
||||
mixed_outputs = _generate(llm, [lora_2d, lora_3d])
|
||||
|
||||
assert mixed_outputs[0] == outputs_2d_alone[0], (
|
||||
f"Mixed-batch 2D output {mixed_outputs[0]!r} does not match "
|
||||
f"standalone 2D output {outputs_2d_alone[0]!r}"
|
||||
)
|
||||
assert mixed_outputs[1] == outputs_3d_alone[1], (
|
||||
f"Mixed-batch 3D output {mixed_outputs[1]!r} does not match "
|
||||
f"standalone 3D output {outputs_3d_alone[1]!r}"
|
||||
)
|
||||
|
||||
# Reverse assignment: neither adapter should be silently aliased.
|
||||
swapped_outputs = _generate(llm, [lora_3d, lora_2d])
|
||||
assert swapped_outputs[0] == outputs_3d_alone[0], (
|
||||
f"Swapped-batch 3D output {swapped_outputs[0]!r} does not match "
|
||||
f"standalone 3D output {outputs_3d_alone[0]!r}"
|
||||
)
|
||||
assert swapped_outputs[1] == outputs_2d_alone[1], (
|
||||
f"Swapped-batch 2D output {swapped_outputs[1]!r} does not match "
|
||||
f"standalone 2D output {outputs_2d_alone[1]!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="This model is too big, so skip this test temporarily.")
|
||||
@pytest.mark.parametrize("fully_sharded_loras", [False, True])
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_qwen36_moe_mixed_2d_3d_lora_tp2(
|
||||
qwen36_moe_2d_lora_files,
|
||||
qwen36_moe_3d_lora_files,
|
||||
fully_sharded_loras,
|
||||
):
|
||||
_run_mixed_2d_3d_lora_test(
|
||||
lora_2d_files=qwen36_moe_2d_lora_files,
|
||||
lora_3d_files=qwen36_moe_3d_lora_files,
|
||||
tensor_parallel_size=2,
|
||||
fully_sharded_loras=fully_sharded_loras,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="This model is too big, so skip this test temporarily.")
|
||||
@pytest.mark.parametrize("fully_sharded_loras", [False, True])
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_qwen36_moe_mixed_2d_3d_lora_tp4(
|
||||
qwen36_moe_2d_lora_files,
|
||||
qwen36_moe_3d_lora_files,
|
||||
fully_sharded_loras,
|
||||
):
|
||||
_run_mixed_2d_3d_lora_test(
|
||||
lora_2d_files=qwen36_moe_2d_lora_files,
|
||||
lora_3d_files=qwen36_moe_3d_lora_files,
|
||||
tensor_parallel_size=4,
|
||||
fully_sharded_loras=fully_sharded_loras,
|
||||
)
|
||||
@@ -77,6 +77,7 @@ def test_qwen3moe_lora(qwen3moe_lora_files):
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
enable_chunked_prefill=True,
|
||||
enable_mixed_moe_lora_format=True,
|
||||
)
|
||||
|
||||
generate_and_test(llm, qwen3moe_lora_files, lora_id=1)
|
||||
@@ -85,7 +86,8 @@ def test_qwen3moe_lora(qwen3moe_lora_files):
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("ep", [False, True])
|
||||
def test_qwen3moe_lora_tp2(ep, qwen3moe_lora_files):
|
||||
@pytest.mark.parametrize("enable_mixed_moe_lora_format", [False, True])
|
||||
def test_qwen3moe_lora_tp2(ep, qwen3moe_lora_files, enable_mixed_moe_lora_format):
|
||||
llm = vllm.LLM(
|
||||
MODEL_PATH,
|
||||
max_model_len=1024,
|
||||
@@ -94,6 +96,7 @@ def test_qwen3moe_lora_tp2(ep, qwen3moe_lora_files):
|
||||
trust_remote_code=True,
|
||||
enable_expert_parallel=ep,
|
||||
tensor_parallel_size=2,
|
||||
enable_mixed_moe_lora_format=enable_mixed_moe_lora_format,
|
||||
)
|
||||
|
||||
generate_and_test(llm, qwen3moe_lora_files, lora_id=1)
|
||||
@@ -102,7 +105,8 @@ def test_qwen3moe_lora_tp2(ep, qwen3moe_lora_files):
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
@pytest.mark.parametrize("ep", [False, True])
|
||||
def test_qwen3moe_lora_tp4(ep, qwen3moe_lora_files):
|
||||
@pytest.mark.parametrize("enable_mixed_moe_lora_format", [False, True])
|
||||
def test_qwen3moe_lora_tp4(ep, qwen3moe_lora_files, enable_mixed_moe_lora_format):
|
||||
llm = vllm.LLM(
|
||||
MODEL_PATH,
|
||||
max_model_len=1024,
|
||||
@@ -111,6 +115,7 @@ def test_qwen3moe_lora_tp4(ep, qwen3moe_lora_files):
|
||||
trust_remote_code=True,
|
||||
enable_expert_parallel=ep,
|
||||
tensor_parallel_size=4,
|
||||
enable_mixed_moe_lora_format=enable_mixed_moe_lora_format,
|
||||
)
|
||||
|
||||
generate_and_test(llm, qwen3moe_lora_files, lora_id=1)
|
||||
|
||||
@@ -71,6 +71,12 @@ class LoRAConfig:
|
||||
for variable LoRA usage patterns at the cost of increased startup time and
|
||||
memory usage. Only takes effect when cudagraph_specialize_lora is True.
|
||||
"""
|
||||
enable_mixed_moe_lora_format: bool = False
|
||||
"""If True, force the engine to use the universal 2D MoE LoRA wrapper
|
||||
(`FusedMoEWithLoRA`) regardless of the model's `is_3d_moe_weight` flag, so
|
||||
that 2D-format and 3D-format MoE LoRA adapters can be served in the same
|
||||
deployment. Only meaningful forMoE models; ignored otherwise. Default False
|
||||
keeps the existing model-driven behavior."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
@@ -90,6 +96,7 @@ class LoRAConfig:
|
||||
factors.append(self.fully_sharded_loras)
|
||||
factors.append(self.lora_dtype)
|
||||
factors.append(self.enable_tower_connector_lora)
|
||||
factors.append(self.enable_mixed_moe_lora_format)
|
||||
# target_modules affects which modules get LoRA applied
|
||||
factors.append(
|
||||
tuple(sorted(self.target_modules)) if self.target_modules else None
|
||||
|
||||
@@ -584,6 +584,7 @@ class EngineArgs:
|
||||
lora_target_modules: list[str] | None = LoRAConfig.target_modules
|
||||
enable_tower_connector_lora: bool = LoRAConfig.enable_tower_connector_lora
|
||||
specialize_active_lora: bool = LoRAConfig.specialize_active_lora
|
||||
enable_mixed_moe_lora_format: bool = LoRAConfig.enable_mixed_moe_lora_format
|
||||
|
||||
ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight
|
||||
num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override
|
||||
@@ -1272,6 +1273,10 @@ class EngineArgs:
|
||||
lora_group.add_argument(
|
||||
"--specialize-active-lora", **lora_kwargs["specialize_active_lora"]
|
||||
)
|
||||
lora_group.add_argument(
|
||||
"--enable-mixed-moe-lora-format",
|
||||
**lora_kwargs["enable_mixed_moe_lora_format"],
|
||||
)
|
||||
|
||||
# Observability arguments
|
||||
observability_kwargs = get_kwargs(ObservabilityConfig)
|
||||
@@ -2009,6 +2014,7 @@ class EngineArgs:
|
||||
target_modules=self.lora_target_modules,
|
||||
enable_tower_connector_lora=self.enable_tower_connector_lora,
|
||||
specialize_active_lora=self.specialize_active_lora,
|
||||
enable_mixed_moe_lora_format=self.enable_mixed_moe_lora_format,
|
||||
max_cpu_loras=self.max_cpu_loras
|
||||
if self.max_cpu_loras and self.max_cpu_loras > 0
|
||||
else None,
|
||||
|
||||
@@ -16,3 +16,4 @@ class LoRAModulePath:
|
||||
name: str
|
||||
path: str
|
||||
base_model_name: str | None = None
|
||||
is_3d_lora_weight: bool = False
|
||||
|
||||
@@ -121,7 +121,9 @@ class OpenAIServingModels:
|
||||
return
|
||||
for lora in self.static_lora_modules:
|
||||
load_request = LoadLoRAAdapterRequest(
|
||||
lora_path=lora.path, lora_name=lora.name
|
||||
lora_path=lora.path,
|
||||
lora_name=lora.name,
|
||||
is_3d_lora_weight=lora.is_3d_lora_weight,
|
||||
)
|
||||
load_result = await self.load_lora_adapter(
|
||||
request=load_request, base_model_name=lora.base_model_name
|
||||
@@ -177,6 +179,7 @@ class OpenAIServingModels:
|
||||
lora_int_id=lora_int_id,
|
||||
lora_path=lora_path,
|
||||
load_inplace=request.load_inplace,
|
||||
is_3d_lora_weight=request.is_3d_lora_weight,
|
||||
)
|
||||
if base_model_name is not None and self.is_base_model(base_model_name):
|
||||
lora_request.base_model_name = base_model_name
|
||||
|
||||
@@ -37,6 +37,7 @@ def attach_router(app: FastAPI):
|
||||
"lora_name": "body.name",
|
||||
"lora_path": "body.src",
|
||||
"load_inplace": "body.load_inplace || `false`",
|
||||
"is_3d_lora_weight": "body.is_3d_lora_weight || `false`",
|
||||
},
|
||||
)
|
||||
@router.post("/v1/load_lora_adapter", dependencies=[Depends(validate_json_request)])
|
||||
|
||||
@@ -8,6 +8,7 @@ class LoadLoRAAdapterRequest(BaseModel):
|
||||
lora_name: str
|
||||
lora_path: str
|
||||
load_inplace: bool = False
|
||||
is_3d_lora_weight: bool = False
|
||||
|
||||
|
||||
class UnloadLoRAAdapterRequest(BaseModel):
|
||||
|
||||
@@ -37,6 +37,10 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
# For non-gated MoE (is_act_and_mul=False), only 1 slice is needed
|
||||
# since there's only up_proj (w1), not gate_proj + up_proj (w1 + w3)
|
||||
self._w13_slices = 2 if base_layer.moe_config.is_act_and_mul else 1
|
||||
# Mirrors per-(lora_id) layout of `self.lora_a_stacked` (built in
|
||||
# `create_lora_weights`) so `create_dummy_lora`'s n_slices fallback
|
||||
# matches `lora_a_stacked` length under EP.
|
||||
self.n_slices = base_layer.local_num_experts * (self._w13_slices + 1)
|
||||
|
||||
self.base_layer.ensure_moe_quant_config_init()
|
||||
if getattr(self.base_layer.quant_method, "supports_internal_mk", False):
|
||||
|
||||
@@ -29,12 +29,17 @@ class LoRAModel:
|
||||
lora_model_id: int,
|
||||
rank: int,
|
||||
loras: dict[str, LoRALayerWeights],
|
||||
is_3d_lora_weight: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
lora_model_id: The integer id for the lora model.
|
||||
rank: lora rank.
|
||||
loras: module name -> weights for lora-replaced layers.
|
||||
is_3d_lora_weight: Whether the on-disk MoE adapter is in the 3D
|
||||
fused (gate_up_proj / down_proj) layout. Propagated from the
|
||||
originating LoRARequest. Only consulted by the LoRA model
|
||||
manager when enable_mixed_moe_lora_format is on.
|
||||
|
||||
"""
|
||||
self.id = lora_model_id
|
||||
@@ -44,6 +49,7 @@ class LoRAModel:
|
||||
)
|
||||
self.rank = rank
|
||||
self.loras: dict[str, LoRALayerWeights] = loras
|
||||
self.is_3d_lora_weight = is_3d_lora_weight
|
||||
|
||||
def clone(self, lora_model_id: int) -> "LoRAModel":
|
||||
"""Return a copy of the object with different ids.
|
||||
@@ -53,6 +59,7 @@ class LoRAModel:
|
||||
lora_model_id,
|
||||
rank=self.rank,
|
||||
loras=self.loras.copy(),
|
||||
is_3d_lora_weight=self.is_3d_lora_weight,
|
||||
)
|
||||
|
||||
def get_lora(self, module_name: str) -> LoRALayerWeights | None:
|
||||
|
||||
+117
-3
@@ -102,7 +102,6 @@ class LoRAModelManager:
|
||||
self.max_num_batched_tokens = math.ceil(max_num_batched_tokens / 8) * 8
|
||||
self.lora_index_to_id: list[int | None] = [None] * self.lora_slots
|
||||
self.vocab_size = vocab_size
|
||||
self.packed_modules_mapping = process_packed_modules_mapping(self.model)
|
||||
|
||||
self.is_pooling_model = is_pooling_model(self.model)
|
||||
self.packed_modules: dict[str, list[str]] = {}
|
||||
@@ -110,7 +109,20 @@ class LoRAModelManager:
|
||||
# Dict instead of a set for compatibility with LRUCache.
|
||||
self._last_mapping: LoRAMapping | None = None
|
||||
is_moe = is_moe_model(self.model)
|
||||
self._is_3d_moe_model = is_moe and self.model.is_3d_moe_weight
|
||||
# Whether the underlying model class declares 3D fused MoE weights.
|
||||
self._model_is_3d_moe = is_moe and self.model.is_3d_moe_weight
|
||||
# When the engine is started with enable_mixed_moe_lora_format=True
|
||||
# we force the universal 2D wrapper (FusedMoEWithLoRA) regardless of
|
||||
# the model's 3D flag, so 2D and 3D adapters can coexist.
|
||||
self._enable_mixed_moe_lora_format = (
|
||||
is_moe and lora_config.enable_mixed_moe_lora_format
|
||||
)
|
||||
self._is_3d_moe_model = (
|
||||
self._model_is_3d_moe and not self._enable_mixed_moe_lora_format
|
||||
)
|
||||
self.packed_modules_mapping = process_packed_modules_mapping(
|
||||
self.model, force_2d_moe=self._enable_mixed_moe_lora_format
|
||||
)
|
||||
self._is_non_gated_moe = is_moe and self.model.is_non_gated_moe
|
||||
self._init_punica_wrapper(max_num_batched_tokens, vllm_config)
|
||||
self._create_lora_modules()
|
||||
@@ -748,7 +760,16 @@ class LoRAModelManager:
|
||||
if isinstance(module, FusedMoE3DWithLoRA):
|
||||
self._stack_moe_lora_weights(lora_model, module, module_name)
|
||||
elif isinstance(module, FusedMoEWithLoRA):
|
||||
self._slice_moe_lora_ep(lora_model, module, module_name)
|
||||
# When mixed mode is enabled the universal 2D wrapper has to
|
||||
# absorb both 2D and 3D-format adapters. 3D-format adapters
|
||||
# need to be split into per-(w1, w2, w3) tensors before the
|
||||
# 2D set_lora can copy them into the stacked buffers.
|
||||
if self._enable_mixed_moe_lora_format and getattr(
|
||||
lora_model, "is_3d_lora_weight", False
|
||||
):
|
||||
self._convert_3d_to_2d_moe_lora(lora_model, module, module_name)
|
||||
else:
|
||||
self._slice_moe_lora_ep(lora_model, module, module_name)
|
||||
|
||||
first_lora: LoRALayerWeights = next(iter(lora_model.loras.values()))
|
||||
assert first_lora.lora_a is not None
|
||||
@@ -871,6 +892,99 @@ class LoRAModelManager:
|
||||
module_lora.lora_a = lora_a
|
||||
module_lora.lora_b = lora_b
|
||||
|
||||
def _convert_3d_to_2d_moe_lora(
|
||||
self,
|
||||
lora_model: LoRAModel,
|
||||
module: FusedMoEWithLoRA,
|
||||
module_name: str,
|
||||
) -> None:
|
||||
"""Convert a 3D-format MoE LoRA checkpoint into the 2D pack layout
|
||||
that `FusedMoEWithLoRA.set_lora` expects.
|
||||
|
||||
On disk the 3D PEFT layout stores two flat tensor pairs per layer:
|
||||
- `{module}.base_layer.lora_{A,B}`: gate_up_proj, with shapes
|
||||
`(rank * num_experts, hidden)` / `(intermediate * 2,
|
||||
rank * num_experts)`
|
||||
- `{module}.lora_{A,B}`: down_proj, with shapes
|
||||
`(rank * num_experts, intermediate)` / `(hidden,
|
||||
rank * num_experts)`
|
||||
The 2D wrapper expects three stacked per-expert tensors,
|
||||
`[w1, w2, w3]`, with `(num_experts, rank, in)` for lora_a and
|
||||
`(num_experts, out, rank)` for lora_b. In the 3D layout w1
|
||||
(gate_proj) and w3 (up_proj) share the rank-r intermediate
|
||||
representation, so both halves use the same lora_a tensor.
|
||||
|
||||
Only invoked when `enable_mixed_moe_lora_format=True` and the
|
||||
source LoRARequest declares `is_3d_lora_weight=True`.
|
||||
"""
|
||||
gate_up_proj_lora = self._get_lora_layer_weights(
|
||||
lora_model, module_name + ".base_layer"
|
||||
)
|
||||
down_proj_lora = self._get_lora_layer_weights(lora_model, module_name)
|
||||
if gate_up_proj_lora is None or down_proj_lora is None:
|
||||
# Either the adapter omits the experts entirely or the file
|
||||
# layout differs from what this path supports; leave the entry
|
||||
# untouched so set_lora can raise a clear error if needed.
|
||||
return
|
||||
|
||||
local_num_experts = module.base_layer.local_num_experts
|
||||
global_num_experts = module.base_layer.global_num_experts
|
||||
ep_rank = module.base_layer.ep_rank
|
||||
expert_start = ep_rank * local_num_experts
|
||||
expert_end = expert_start + local_num_experts
|
||||
|
||||
# Reshape and EP-slice into per-expert 3D tensors. This mirrors
|
||||
# `_stack_moe_lora_weights`; for non-EP runs the slice is a no-op.
|
||||
gate_up_a = gate_up_proj_lora.lora_a.reshape(
|
||||
global_num_experts, -1, gate_up_proj_lora.lora_a.shape[-1]
|
||||
)[expert_start:expert_end].contiguous()
|
||||
gate_up_b = (
|
||||
gate_up_proj_lora.lora_b.reshape(
|
||||
gate_up_proj_lora.lora_b.shape[0], -1, global_num_experts
|
||||
)[..., expert_start:expert_end]
|
||||
.permute(2, 0, 1)
|
||||
.contiguous()
|
||||
)
|
||||
down_a = down_proj_lora.lora_a.reshape(
|
||||
global_num_experts, -1, down_proj_lora.lora_a.shape[-1]
|
||||
)[expert_start:expert_end].contiguous()
|
||||
down_b = (
|
||||
down_proj_lora.lora_b.reshape(
|
||||
down_proj_lora.lora_b.shape[0], -1, global_num_experts
|
||||
)[..., expert_start:expert_end]
|
||||
.permute(2, 0, 1)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
# Split the fused gate_up_proj output dim into separate w1 / w3
|
||||
# halves. GPT-OSS interleaves them along the output dim, all other
|
||||
# 3D MoE checkpoints we know about concatenate them.
|
||||
intermediate_x2 = gate_up_b.shape[1]
|
||||
if intermediate_x2 % 2 != 0:
|
||||
raise ValueError(
|
||||
"Expected gate_up_proj LoRA-B output dim to be 2 * intermediate, "
|
||||
f"got {intermediate_x2}."
|
||||
)
|
||||
intermediate = intermediate_x2 // 2
|
||||
base_arch = self.model.config.architectures[0]
|
||||
if base_arch == "GptOssForCausalLM":
|
||||
w1_b = gate_up_b[:, ::2, :].contiguous()
|
||||
w3_b = gate_up_b[:, 1::2, :].contiguous()
|
||||
else:
|
||||
w1_b = gate_up_b[:, :intermediate, :].contiguous()
|
||||
w3_b = gate_up_b[:, intermediate:, :].contiguous()
|
||||
|
||||
# In the 3D layout w1 and w3 share the same rank-r mid
|
||||
# representation, so they reuse the same lora_a tensor. The 2D
|
||||
# wrapper's set_lora copies whatever it gets here into independent
|
||||
# per-slice buffers, so the sharing is purely a CPU-side memory
|
||||
# optimization and does not affect numerics.
|
||||
down_proj_lora.lora_a = [gate_up_a, down_a, gate_up_a]
|
||||
down_proj_lora.lora_b = [w1_b, down_b, w3_b]
|
||||
# Drop the redundant base_layer entry to avoid double pin_memory
|
||||
# and to keep the activation path looking up only the wrapper key.
|
||||
lora_model.loras.pop(module_name + ".base_layer", None)
|
||||
|
||||
def _slice_moe_lora_ep(
|
||||
self,
|
||||
lora_model: LoRAModel,
|
||||
|
||||
@@ -28,6 +28,13 @@ class LoRARequest(
|
||||
base_model_name: str | None = msgspec.field(default=None)
|
||||
tensorizer_config_dict: dict | None = None
|
||||
load_inplace: bool = False
|
||||
is_3d_lora_weight: bool = False
|
||||
"""Whether this adapter's MoE weights are stored in the 3D fused
|
||||
`gate_up_proj` / `down_proj` layout (one fused tensor per layer) or the
|
||||
2D per-expert split layout (separate `gate_proj` / `up_proj` / `down_proj`
|
||||
tensors per expert). Only consulted when the engine is started with
|
||||
`enable_mixed_moe_lora_format=True`; otherwise it is ignored and the
|
||||
on-disk format is inferred from the base model."""
|
||||
|
||||
def __post_init__(self):
|
||||
if self.lora_int_id < 1:
|
||||
|
||||
+8
-3
@@ -355,7 +355,9 @@ def get_adapter_absolute_path(lora_path: str) -> str:
|
||||
return local_snapshot_path
|
||||
|
||||
|
||||
def process_packed_modules_mapping(model: nn.Module) -> dict[str, list[str]]:
|
||||
def process_packed_modules_mapping(
|
||||
model: nn.Module, force_2d_moe: bool = False
|
||||
) -> dict[str, list[str]]:
|
||||
if is_moe_model(model):
|
||||
if moe_packed_mapping := get_moe_expert_mapping(model):
|
||||
# This method generates and returns a dictionary mapping packed module
|
||||
@@ -364,8 +366,11 @@ def process_packed_modules_mapping(model: nn.Module) -> dict[str, list[str]]:
|
||||
# the expert indices are expanded based on the configured number
|
||||
# of routed experts.
|
||||
packed_modules_mapping = get_packed_modules_mapping(model)
|
||||
if not model.is_3d_moe_weight:
|
||||
# 3D MoE LoRA does not need `packed_modules_mapping`
|
||||
# The 2D mapping is needed when the model itself is 2D, or when
|
||||
# the engine forces the universal 2D wrapper via
|
||||
# enable_mixed_moe_lora_format (so 3D models can also load 2D
|
||||
# adapters through FusedMoEWithLoRA).
|
||||
if (not model.is_3d_moe_weight) or force_2d_moe:
|
||||
# Filter out malformed entries: non-gated MoE has empty
|
||||
# ckpt_up_proj_name which results in weight_name containing ".."
|
||||
# (e.g., "experts.0.." instead of "experts.0.layer_name.")
|
||||
|
||||
@@ -141,6 +141,10 @@ class WorkerLoRAManager:
|
||||
weights_mapper=hf_to_vllm_mapper,
|
||||
skip_prefixes=lora_skip_prefixes,
|
||||
)
|
||||
# Stamp the on-disk MoE layout onto the loaded model so the
|
||||
# adapter manager can route 3D-format checkpoints through the
|
||||
# 3D->2D conversion when running under the universal 2D wrapper.
|
||||
lora.is_3d_lora_weight = lora_request.is_3d_lora_weight
|
||||
|
||||
except FileNotFoundError as e:
|
||||
# FileNotFoundError should be raised if both
|
||||
|
||||
Reference in New Issue
Block a user