[Neuron] Add multi-LoRA support for Neuron. (#18284)
Signed-off-by: Satyajith Chilappagari <satchill@amazon.com>
This commit is contained in:
@@ -1,17 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, Set
|
||||
|
||||
import torch
|
||||
from neuronx_distributed_inference.modules.generation.sampling import (
|
||||
prepare_sampling_params)
|
||||
from neuronx_distributed_inference.modules.lora_serving import (
|
||||
LoraCheckpoint, LoraServingConfig)
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.entrypoints.openai.serving_models import LoRAModulePath
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.layers import LoRAMapping
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor import SamplingMetadata
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput
|
||||
from vllm.model_executor.model_loader.neuronx_distributed import (
|
||||
_get_model_architecture, get_neuron_model)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
|
||||
from vllm.worker.neuron_model_runner import (ModelInputForNeuron,
|
||||
NeuronModelRunner)
|
||||
|
||||
@@ -25,11 +32,44 @@ class NeuronxDistributedModelRunner(NeuronModelRunner):
|
||||
vllm_config: VllmConfig,
|
||||
):
|
||||
super().__init__(vllm_config)
|
||||
self.lora_checkpoint = None
|
||||
self.model = None
|
||||
self.lora_serving_config = None
|
||||
|
||||
@staticmethod
|
||||
def _get_lora_paths_strings(lora_modules: List[LoRAModulePath]):
|
||||
if not lora_modules:
|
||||
return None
|
||||
return {_.get("name"): _.get("path") for _ in lora_modules}
|
||||
|
||||
def _get_nxdi_lora_config(self):
|
||||
override_neuron_config = self.model_config.override_neuron_config
|
||||
lora_modules = override_neuron_config.pop("lora_modules", None)
|
||||
target_modules = override_neuron_config.pop("target_modules", None)
|
||||
lora_ckpt_paths = self._get_lora_paths_strings(lora_modules)
|
||||
if self.lora_config.max_loras < len(lora_ckpt_paths):
|
||||
raise ValueError(
|
||||
"Number of LoRAs (%s) exceeds maximum "
|
||||
"allowed (%s)", len(lora_ckpt_paths),
|
||||
self.lora_config.max_loras)
|
||||
|
||||
return LoraServingConfig(
|
||||
max_loras=self.lora_config.max_loras,
|
||||
max_lora_rank=self.lora_config.max_lora_rank,
|
||||
target_modules=target_modules,
|
||||
lora_ckpt_paths=lora_ckpt_paths,
|
||||
)
|
||||
|
||||
def load_model(self) -> None:
|
||||
self.model = get_neuron_model(self.model_config,
|
||||
parallel_config=self.parallel_config,
|
||||
scheduler_config=self.scheduler_config)
|
||||
# Update LoRA config
|
||||
if self.lora_config is not None:
|
||||
self.lora_serving_config = self._get_nxdi_lora_config()
|
||||
self.lora_checkpoint = LoraCheckpoint(self.lora_serving_config)
|
||||
self.model = get_neuron_model(
|
||||
self.model_config,
|
||||
parallel_config=self.parallel_config,
|
||||
scheduler_config=self.scheduler_config,
|
||||
lora_serving_config=self.lora_serving_config)
|
||||
|
||||
def get_nxd_sampling_params(self, sampling_metadata):
|
||||
if self.model.config.neuron_config.on_device_sampling_config:
|
||||
@@ -134,3 +174,116 @@ class NeuronxDistributedModelRunner(NeuronModelRunner):
|
||||
)
|
||||
|
||||
return [output]
|
||||
|
||||
def _get_lora_adapter_ids(self, seq_group_metadata_list):
|
||||
# set LoRA adapter IDs for multi-lora serving
|
||||
batch_size = len(seq_group_metadata_list)
|
||||
if self.lora_checkpoint is not None:
|
||||
# "0" indicates NxDI to use the base model for inference
|
||||
adapter_ids = ["0"] * batch_size
|
||||
for idx, seq_group_metadata in enumerate(seq_group_metadata_list):
|
||||
if seq_group_metadata.lora_request is not None:
|
||||
adapter_ids[
|
||||
idx] = seq_group_metadata.lora_request.lora_name
|
||||
|
||||
# convert adapter_ids from strings to integers
|
||||
adapter_ids = self.lora_checkpoint.convert_adapter_ids_to_indices(
|
||||
adapter_ids, batch_size)
|
||||
else:
|
||||
adapter_ids = torch.zeros((batch_size), dtype=torch.int32)
|
||||
|
||||
return adapter_ids
|
||||
|
||||
def prepare_model_input(
|
||||
self,
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
virtual_engine: int = 0,
|
||||
finished_requests_ids: Optional[List[str]] = None
|
||||
) -> ModelInputForNeuron:
|
||||
multi_modal_kwargs = None
|
||||
# NOTE: We assume that all sequences in the group are all prompts or
|
||||
# all decodes.
|
||||
is_prompt = seq_group_metadata_list[0].is_prompt
|
||||
# Prepare input tensors.
|
||||
if is_prompt:
|
||||
(input_tokens, input_positions, input_block_ids, seq_lens,
|
||||
multi_modal_kwargs
|
||||
) = self._prepare_prompt(seq_group_metadata_list)
|
||||
else:
|
||||
(input_tokens, input_positions,
|
||||
input_block_ids) = self._prepare_decode(seq_group_metadata_list)
|
||||
seq_lens = None
|
||||
|
||||
if not self._on_device_sampling_disabled:
|
||||
for seq_group_metadata in seq_group_metadata_list:
|
||||
sampling_params = seq_group_metadata.sampling_params
|
||||
top_k, top_p, temperature = (
|
||||
self._convert_to_neuron_sampling_params(sampling_params))
|
||||
sampling_params.top_k = top_k
|
||||
sampling_params.top_p = top_p
|
||||
sampling_params.temperature = temperature
|
||||
|
||||
lora_adapter_ids = self._get_lora_adapter_ids(seq_group_metadata_list)
|
||||
|
||||
sampling_metadata = SamplingMetadata.prepare(
|
||||
seq_group_metadata_list,
|
||||
seq_lens,
|
||||
# query_lens is not needed if chunked prefill is not
|
||||
# supported. Since neuron worker doesn't support chunked prefill
|
||||
# just use seq_lens instead.
|
||||
seq_lens,
|
||||
self.device,
|
||||
self.pin_memory,
|
||||
generators=self.get_generators(finished_requests_ids))
|
||||
|
||||
if current_platform.use_transformers_neuronx(
|
||||
) and not self._on_device_sampling_disabled:
|
||||
# Once the request IDs are changed in current iteration, we will
|
||||
# update the on-device sampling parameters.
|
||||
current_batch_request_ids = [
|
||||
seq_group_meta_data.request_id
|
||||
for seq_group_meta_data in seq_group_metadata_list
|
||||
]
|
||||
if current_batch_request_ids != self._previous_batch_request_ids:
|
||||
self._update_neuron_sampling_params(seq_group_metadata_list)
|
||||
self._previous_batch_request_ids = current_batch_request_ids
|
||||
|
||||
return ModelInputForNeuron(input_tokens=input_tokens,
|
||||
input_positions=input_positions,
|
||||
input_block_ids=input_block_ids,
|
||||
sampling_metadata=sampling_metadata,
|
||||
multi_modal_kwargs=multi_modal_kwargs,
|
||||
adapter_ids=lora_adapter_ids)
|
||||
|
||||
def remove_all_loras(self):
|
||||
raise NotImplementedError(
|
||||
"Managing LoRAs is only supported through the "
|
||||
"lora_modules parameter in override_neuron_config")
|
||||
|
||||
def set_active_loras(self, lora_requests: Set[LoRARequest],
|
||||
lora_mapping: LoRAMapping) -> None:
|
||||
raise NotImplementedError(
|
||||
"Managing LoRAs is only supported through the "
|
||||
"lora_modules parameter in override_neuron_config")
|
||||
|
||||
def add_lora(self, lora_request: LoRARequest):
|
||||
logger.warning(
|
||||
"Adding LoRAs is only supported through the "
|
||||
"lora_modules parameter in override_neuron_config. If you supplied "
|
||||
"the parameter, you can ignore this warning. Ignoring"
|
||||
"lora request: ", lora_request)
|
||||
|
||||
def remove_lora(self, lora_id: int) -> bool:
|
||||
raise NotImplementedError(
|
||||
"Managing LoRAs is only supported through the "
|
||||
"lora_modules parameter in override_neuron_config")
|
||||
|
||||
def pin_lora(self, lora_id: int) -> bool:
|
||||
raise NotImplementedError(
|
||||
"Managing LoRAs is only supported through the "
|
||||
"lora_modules parameter in override_neuron_config")
|
||||
|
||||
def list_loras(self) -> Set[int]:
|
||||
raise NotImplementedError(
|
||||
"Managing LoRAs is only supported through the "
|
||||
"lora_modules parameter in override_neuron_config")
|
||||
|
||||
Reference in New Issue
Block a user