[Core] Add Support for Default Modality Specific LoRAs [generate / chat completions] (#19126)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
This commit is contained in:
Alex Brooks
2025-07-10 21:09:37 +01:00
committed by GitHub
parent 3de2ed767f
commit 41060c6e08
9 changed files with 482 additions and 5 deletions
+81
View File
@@ -499,6 +499,10 @@ class LLM:
_validate_truncation_size(self.llm_engine.model_config.max_model_len,
truncate_prompt_tokens, tokenization_kwargs)
# Add any modality specific loras to the corresponding prompts
lora_request = self._get_modality_specific_lora_reqs(
parsed_prompts, lora_request)
self._validate_and_add_requests(
prompts=parsed_prompts,
params=sampling_params,
@@ -513,6 +517,83 @@ class LLM:
outputs = self._run_engine(use_tqdm=use_tqdm)
return self.engine_class.validate_outputs(outputs, RequestOutput)
def _get_modality_specific_lora_reqs(
self, parsed_prompts: Union[PromptType, Sequence[PromptType]],
lora_request: Optional[Union[list[LoRARequest], LoRARequest]]):
# Grab the lora config off the vllm config on the engine,
# since this is the same for both v0 & v1.
lora_config = self.llm_engine.vllm_config.lora_config
# If there's no lora config / default_mm_loras, or the model
# isn't multimodal, leave the lora as is.
if (lora_config is None
or not self.llm_engine.model_config.is_multimodal_model
or (lora_config and lora_config.default_mm_loras is None)):
return lora_request
if not isinstance(parsed_prompts, Sequence):
parsed_prompts = [parsed_prompts]
optional_loras = ([lora_request] * len(parsed_prompts)
if not isinstance(lora_request, Sequence) else
lora_request)
return [
self._resolve_single_prompt_mm_lora(
parsed_prompt,
opt_lora_req,
lora_config.default_mm_loras,
) for parsed_prompt, opt_lora_req in zip(parsed_prompts,
optional_loras)
]
def _resolve_single_prompt_mm_lora(self, parsed_prompt: PromptType,
lora_request: Optional[LoRARequest],
default_mm_loras: Optional[dict[str,
str]]):
if (not default_mm_loras or not isinstance(parsed_prompt, dict)
or "multi_modal_data" not in parsed_prompt):
return lora_request
parsed_prompt = cast(Union[TextPrompt, TokensPrompt], parsed_prompt)
intersection = set(
parsed_prompt["multi_modal_data"].keys()).intersection(
default_mm_loras.keys())
if not intersection:
return lora_request
if len(intersection) > 1:
# TODO: Would be nice to be able to have multiple loras per prompt
logger.warning(
"Multiple modality specific loras were registered and would be"
" used by a single prompt consuming several modalities; "
" currently we only support one lora per request; as such,"
" lora(s) registered with modalities: %s"
" will be skipped", intersection)
return lora_request
# Build the LoRA request; the ID of the default mm lora is the
# index of the modality name sorted alphabetically + 1.
modality_name = intersection.pop()
modality_lora_path = default_mm_loras[modality_name]
modality_lora_id = sorted(default_mm_loras).index(modality_name) + 1
# If we have a collision, warn if there is a collision,
# but always send the explicitly provided request.
if lora_request:
if lora_request.lora_int_id != modality_lora_id:
logger.warning(
"A modality with a registered lora and a lora_request "
"with a different ID were provided; falling back to the "
"lora_request as we only apply one LoRARequest per prompt")
return lora_request
return LoRARequest(
modality_name,
modality_lora_id,
modality_lora_path,
)
def collective_rpc(self,
method: Union[str, Callable[..., _R]],
timeout: Optional[float] = None,
+19 -1
View File
@@ -87,6 +87,7 @@ from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
from vllm.entrypoints.openai.serving_engine import OpenAIServing
from vllm.entrypoints.openai.serving_models import (BaseModelPath,
LoRAModulePath,
OpenAIServingModels)
from vllm.entrypoints.openai.serving_pooling import OpenAIServingPooling
from vllm.entrypoints.openai.serving_responses import OpenAIServingResponses
@@ -1481,11 +1482,28 @@ async def init_app_state(
"This discrepancy may lead to performance degradation.",
resolved_chat_template, args.model)
# Merge default_mm_loras into the static lora_modules
default_mm_loras = (vllm_config.lora_config.default_mm_loras
if vllm_config.lora_config is not None else {})
lora_modules = args.lora_modules
if default_mm_loras:
default_mm_lora_paths = [
LoRAModulePath(
name=modality,
path=lora_path,
) for modality, lora_path in default_mm_loras.items()
]
if args.lora_modules is None:
lora_modules = default_mm_lora_paths
else:
lora_modules += default_mm_lora_paths
state.openai_serving_models = OpenAIServingModels(
engine_client=engine_client,
model_config=model_config,
base_model_paths=base_model_paths,
lora_modules=args.lora_modules,
lora_modules=lora_modules,
prompt_adapters=args.prompt_adapters,
)
await state.openai_serving_models.init_static_loras()
+2 -1
View File
@@ -153,7 +153,8 @@ class OpenAIServingChat(OpenAIServing):
(
lora_request,
prompt_adapter_request,
) = self._maybe_get_adapters(request)
) = self._maybe_get_adapters(request,
supports_default_mm_loras=True)
model_name = self._get_model_name(request.model, lora_request)
+57 -3
View File
@@ -458,20 +458,74 @@ class OpenAIServing:
err_type="NotFoundError",
status_code=HTTPStatus.NOT_FOUND)
def _get_active_default_mm_loras(
self, request: AnyRequest) -> Optional[LoRARequest]:
"""Determine if there are any active default multimodal loras."""
# TODO: Currently this is only enabled for chat completions
# to be better aligned with only being enabled for .generate
# when run offline. It would be nice to support additional
# tasks types in the future.
message_types = self._get_message_types(request)
default_mm_loras = set()
for lora in self.models.lora_requests.values():
# Best effort match for default multimodal lora adapters;
# There is probably a better way to do this, but currently
# this matches against the set of 'types' in any content lists
# up until '_', e.g., to match audio_url -> audio
if lora.lora_name in message_types:
default_mm_loras.add(lora)
# Currently only support default modality specific loras if
# we have exactly one lora matched on the request.
if len(default_mm_loras) == 1:
return default_mm_loras.pop()
return None
def _maybe_get_adapters(
self, request: AnyRequest
self,
request: AnyRequest,
supports_default_mm_loras: bool = False,
) -> Union[tuple[None, None], tuple[LoRARequest, None], tuple[
None, PromptAdapterRequest]]:
if self._is_model_supported(request.model):
return None, None
if request.model in self.models.lora_requests:
return self.models.lora_requests[request.model], None
# Currently only support default modality specific loras
# if we have exactly one lora matched on the request.
if supports_default_mm_loras:
default_mm_lora = self._get_active_default_mm_loras(request)
if default_mm_lora is not None:
return default_mm_lora, None
if self._is_model_supported(request.model):
return None, None
for prompt_adapter in self.models.prompt_adapter_requests:
if request.model == prompt_adapter.prompt_adapter_name:
return None, prompt_adapter
# if _check_model has been called earlier, this will be unreachable
raise ValueError(f"The model `{request.model}` does not exist.")
def _get_message_types(self, request: AnyRequest) -> set[str]:
"""Retrieve the set of types from message content dicts up
until `_`; we use this to match potential multimodal data
with default per modality loras.
"""
message_types: set[str] = set()
if not hasattr(request, "messages"):
return message_types
for message in request.messages:
if (isinstance(message, dict) and "content" in message
and isinstance(message["content"], list)):
for content_dict in message["content"]:
if "type" in content_dict:
message_types.add(content_dict["type"].split("_")[0])
return message_types
async def _normalize_prompt_text_to_input(
self,
request: AnyRequest,