[Core] Add Support for Default Modality Specific LoRAs [generate / chat completions] (#19126)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
This commit is contained in:
@@ -499,6 +499,10 @@ class LLM:
|
||||
_validate_truncation_size(self.llm_engine.model_config.max_model_len,
|
||||
truncate_prompt_tokens, tokenization_kwargs)
|
||||
|
||||
# Add any modality specific loras to the corresponding prompts
|
||||
lora_request = self._get_modality_specific_lora_reqs(
|
||||
parsed_prompts, lora_request)
|
||||
|
||||
self._validate_and_add_requests(
|
||||
prompts=parsed_prompts,
|
||||
params=sampling_params,
|
||||
@@ -513,6 +517,83 @@ class LLM:
|
||||
outputs = self._run_engine(use_tqdm=use_tqdm)
|
||||
return self.engine_class.validate_outputs(outputs, RequestOutput)
|
||||
|
||||
def _get_modality_specific_lora_reqs(
|
||||
self, parsed_prompts: Union[PromptType, Sequence[PromptType]],
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]]):
|
||||
# Grab the lora config off the vllm config on the engine,
|
||||
# since this is the same for both v0 & v1.
|
||||
lora_config = self.llm_engine.vllm_config.lora_config
|
||||
|
||||
# If there's no lora config / default_mm_loras, or the model
|
||||
# isn't multimodal, leave the lora as is.
|
||||
if (lora_config is None
|
||||
or not self.llm_engine.model_config.is_multimodal_model
|
||||
or (lora_config and lora_config.default_mm_loras is None)):
|
||||
return lora_request
|
||||
|
||||
if not isinstance(parsed_prompts, Sequence):
|
||||
parsed_prompts = [parsed_prompts]
|
||||
|
||||
optional_loras = ([lora_request] * len(parsed_prompts)
|
||||
if not isinstance(lora_request, Sequence) else
|
||||
lora_request)
|
||||
|
||||
return [
|
||||
self._resolve_single_prompt_mm_lora(
|
||||
parsed_prompt,
|
||||
opt_lora_req,
|
||||
lora_config.default_mm_loras,
|
||||
) for parsed_prompt, opt_lora_req in zip(parsed_prompts,
|
||||
optional_loras)
|
||||
]
|
||||
|
||||
def _resolve_single_prompt_mm_lora(self, parsed_prompt: PromptType,
|
||||
lora_request: Optional[LoRARequest],
|
||||
default_mm_loras: Optional[dict[str,
|
||||
str]]):
|
||||
if (not default_mm_loras or not isinstance(parsed_prompt, dict)
|
||||
or "multi_modal_data" not in parsed_prompt):
|
||||
return lora_request
|
||||
|
||||
parsed_prompt = cast(Union[TextPrompt, TokensPrompt], parsed_prompt)
|
||||
|
||||
intersection = set(
|
||||
parsed_prompt["multi_modal_data"].keys()).intersection(
|
||||
default_mm_loras.keys())
|
||||
if not intersection:
|
||||
return lora_request
|
||||
if len(intersection) > 1:
|
||||
# TODO: Would be nice to be able to have multiple loras per prompt
|
||||
logger.warning(
|
||||
"Multiple modality specific loras were registered and would be"
|
||||
" used by a single prompt consuming several modalities; "
|
||||
" currently we only support one lora per request; as such,"
|
||||
" lora(s) registered with modalities: %s"
|
||||
" will be skipped", intersection)
|
||||
return lora_request
|
||||
|
||||
# Build the LoRA request; the ID of the default mm lora is the
|
||||
# index of the modality name sorted alphabetically + 1.
|
||||
modality_name = intersection.pop()
|
||||
modality_lora_path = default_mm_loras[modality_name]
|
||||
modality_lora_id = sorted(default_mm_loras).index(modality_name) + 1
|
||||
|
||||
# If we have a collision, warn if there is a collision,
|
||||
# but always send the explicitly provided request.
|
||||
if lora_request:
|
||||
if lora_request.lora_int_id != modality_lora_id:
|
||||
logger.warning(
|
||||
"A modality with a registered lora and a lora_request "
|
||||
"with a different ID were provided; falling back to the "
|
||||
"lora_request as we only apply one LoRARequest per prompt")
|
||||
return lora_request
|
||||
|
||||
return LoRARequest(
|
||||
modality_name,
|
||||
modality_lora_id,
|
||||
modality_lora_path,
|
||||
)
|
||||
|
||||
def collective_rpc(self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
|
||||
@@ -87,6 +87,7 @@ from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
|
||||
from vllm.entrypoints.openai.serving_engine import OpenAIServing
|
||||
from vllm.entrypoints.openai.serving_models import (BaseModelPath,
|
||||
LoRAModulePath,
|
||||
OpenAIServingModels)
|
||||
from vllm.entrypoints.openai.serving_pooling import OpenAIServingPooling
|
||||
from vllm.entrypoints.openai.serving_responses import OpenAIServingResponses
|
||||
@@ -1481,11 +1482,28 @@ async def init_app_state(
|
||||
"This discrepancy may lead to performance degradation.",
|
||||
resolved_chat_template, args.model)
|
||||
|
||||
# Merge default_mm_loras into the static lora_modules
|
||||
default_mm_loras = (vllm_config.lora_config.default_mm_loras
|
||||
if vllm_config.lora_config is not None else {})
|
||||
|
||||
lora_modules = args.lora_modules
|
||||
if default_mm_loras:
|
||||
default_mm_lora_paths = [
|
||||
LoRAModulePath(
|
||||
name=modality,
|
||||
path=lora_path,
|
||||
) for modality, lora_path in default_mm_loras.items()
|
||||
]
|
||||
if args.lora_modules is None:
|
||||
lora_modules = default_mm_lora_paths
|
||||
else:
|
||||
lora_modules += default_mm_lora_paths
|
||||
|
||||
state.openai_serving_models = OpenAIServingModels(
|
||||
engine_client=engine_client,
|
||||
model_config=model_config,
|
||||
base_model_paths=base_model_paths,
|
||||
lora_modules=args.lora_modules,
|
||||
lora_modules=lora_modules,
|
||||
prompt_adapters=args.prompt_adapters,
|
||||
)
|
||||
await state.openai_serving_models.init_static_loras()
|
||||
|
||||
@@ -153,7 +153,8 @@ class OpenAIServingChat(OpenAIServing):
|
||||
(
|
||||
lora_request,
|
||||
prompt_adapter_request,
|
||||
) = self._maybe_get_adapters(request)
|
||||
) = self._maybe_get_adapters(request,
|
||||
supports_default_mm_loras=True)
|
||||
|
||||
model_name = self._get_model_name(request.model, lora_request)
|
||||
|
||||
|
||||
@@ -458,20 +458,74 @@ class OpenAIServing:
|
||||
err_type="NotFoundError",
|
||||
status_code=HTTPStatus.NOT_FOUND)
|
||||
|
||||
def _get_active_default_mm_loras(
|
||||
self, request: AnyRequest) -> Optional[LoRARequest]:
|
||||
"""Determine if there are any active default multimodal loras."""
|
||||
# TODO: Currently this is only enabled for chat completions
|
||||
# to be better aligned with only being enabled for .generate
|
||||
# when run offline. It would be nice to support additional
|
||||
# tasks types in the future.
|
||||
message_types = self._get_message_types(request)
|
||||
default_mm_loras = set()
|
||||
|
||||
for lora in self.models.lora_requests.values():
|
||||
# Best effort match for default multimodal lora adapters;
|
||||
# There is probably a better way to do this, but currently
|
||||
# this matches against the set of 'types' in any content lists
|
||||
# up until '_', e.g., to match audio_url -> audio
|
||||
if lora.lora_name in message_types:
|
||||
default_mm_loras.add(lora)
|
||||
|
||||
# Currently only support default modality specific loras if
|
||||
# we have exactly one lora matched on the request.
|
||||
if len(default_mm_loras) == 1:
|
||||
return default_mm_loras.pop()
|
||||
return None
|
||||
|
||||
def _maybe_get_adapters(
|
||||
self, request: AnyRequest
|
||||
self,
|
||||
request: AnyRequest,
|
||||
supports_default_mm_loras: bool = False,
|
||||
) -> Union[tuple[None, None], tuple[LoRARequest, None], tuple[
|
||||
None, PromptAdapterRequest]]:
|
||||
if self._is_model_supported(request.model):
|
||||
return None, None
|
||||
|
||||
if request.model in self.models.lora_requests:
|
||||
return self.models.lora_requests[request.model], None
|
||||
|
||||
# Currently only support default modality specific loras
|
||||
# if we have exactly one lora matched on the request.
|
||||
if supports_default_mm_loras:
|
||||
default_mm_lora = self._get_active_default_mm_loras(request)
|
||||
if default_mm_lora is not None:
|
||||
return default_mm_lora, None
|
||||
|
||||
if self._is_model_supported(request.model):
|
||||
return None, None
|
||||
|
||||
for prompt_adapter in self.models.prompt_adapter_requests:
|
||||
if request.model == prompt_adapter.prompt_adapter_name:
|
||||
return None, prompt_adapter
|
||||
# if _check_model has been called earlier, this will be unreachable
|
||||
raise ValueError(f"The model `{request.model}` does not exist.")
|
||||
|
||||
def _get_message_types(self, request: AnyRequest) -> set[str]:
|
||||
"""Retrieve the set of types from message content dicts up
|
||||
until `_`; we use this to match potential multimodal data
|
||||
with default per modality loras.
|
||||
"""
|
||||
message_types: set[str] = set()
|
||||
|
||||
if not hasattr(request, "messages"):
|
||||
return message_types
|
||||
|
||||
for message in request.messages:
|
||||
if (isinstance(message, dict) and "content" in message
|
||||
and isinstance(message["content"], list)):
|
||||
for content_dict in message["content"]:
|
||||
if "type" in content_dict:
|
||||
message_types.add(content_dict["type"].split("_")[0])
|
||||
return message_types
|
||||
|
||||
async def _normalize_prompt_text_to_input(
|
||||
self,
|
||||
request: AnyRequest,
|
||||
|
||||
Reference in New Issue
Block a user