[Model Runner V2] Fix mrv2 mm lora issue (#44450)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -24,8 +24,11 @@ from vllm.distributed.parallel_state import (
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.base import SparseWeightPatch
|
||||
from vllm.lora.layers import LoRAMappingType
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
@@ -44,6 +47,9 @@ from vllm.v1.kv_cache_interface import (
|
||||
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||
from vllm.v1.worker.gpu.lora_utils import LoraState
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.lora import set_active_mm_loras
|
||||
from vllm.v1.worker.gpu_input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.utils import select_common_block_size
|
||||
@@ -314,6 +320,79 @@ def test_select_common_block_size_no_valid_option():
|
||||
select_common_block_size(48, [backend_a, backend_b])
|
||||
|
||||
|
||||
def test_set_active_mm_loras_builds_tower_and_connector_mappings():
|
||||
model = Mock()
|
||||
model.get_num_mm_encoder_tokens.side_effect = lambda num_embeds: num_embeds + 1
|
||||
model.get_mm_mapping.return_value = SimpleNamespace(connector=True)
|
||||
model.get_num_mm_connector_tokens.side_effect = lambda num_tokens: num_tokens + 10
|
||||
|
||||
lora_manager = Mock()
|
||||
lora_manager.supports_tower_connector_lora.return_value = True
|
||||
|
||||
encoder_cache = EncoderCache()
|
||||
encoder_cache.mm_features["req-with-lora"] = [
|
||||
MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="image",
|
||||
identifier="img-0",
|
||||
mm_position=PlaceholderRange(offset=0, length=2),
|
||||
),
|
||||
MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="image",
|
||||
identifier="img-1",
|
||||
mm_position=PlaceholderRange(offset=2, length=3),
|
||||
),
|
||||
]
|
||||
encoder_cache.mm_features["req-no-lora"] = [
|
||||
MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="image",
|
||||
identifier="img-2",
|
||||
mm_position=PlaceholderRange(offset=0, length=1),
|
||||
)
|
||||
]
|
||||
|
||||
lora_state = LoraState(max_num_reqs=4)
|
||||
lora_request = LoRARequest("vision-lora", 7, "/tmp/vision-lora")
|
||||
lora_state.add_request("req-with-lora", 0, lora_request)
|
||||
lora_state.add_request("req-no-lora", 1, None)
|
||||
|
||||
set_active_mm_loras(
|
||||
model=model,
|
||||
lora_manager=lora_manager,
|
||||
encoder_cache=encoder_cache,
|
||||
req_id_to_index={
|
||||
"req-with-lora": 0,
|
||||
"req-no-lora": 1,
|
||||
},
|
||||
lora_state=lora_state,
|
||||
scheduled_encoder_inputs={
|
||||
"req-with-lora": [1, 0],
|
||||
"req-no-lora": [0],
|
||||
"missing-req": [0],
|
||||
},
|
||||
)
|
||||
|
||||
assert lora_manager.set_active_adapters.call_count == 2
|
||||
|
||||
tower_requests, tower_mapping = lora_manager.set_active_adapters.call_args_list[
|
||||
0
|
||||
].args
|
||||
assert tower_requests == {lora_request}
|
||||
assert tower_mapping.type is LoRAMappingType.TOWER
|
||||
assert tower_mapping.prompt_mapping == (7, 7, 0)
|
||||
assert tower_mapping.index_mapping == (7, 7, 7, 7, 7, 7, 7, 0, 0)
|
||||
|
||||
connector_requests, connector_mapping = (
|
||||
lora_manager.set_active_adapters.call_args_list[1].args
|
||||
)
|
||||
assert connector_requests == {lora_request}
|
||||
assert connector_mapping.type is LoRAMappingType.CONNECTOR
|
||||
assert connector_mapping.prompt_mapping == (7, 7, 0)
|
||||
assert connector_mapping.index_mapping == ((7,) * 14 + (7,) * 13 + (0,) * 12)
|
||||
|
||||
|
||||
def test_update_states_new_request(model_runner, dist_init):
|
||||
req_id = "req_0"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user