[Model Runner V2] Add LoRAState to consolidate lora logic (#33062)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
@@ -51,6 +51,7 @@ from vllm.v1.worker.gpu.kv_connector import (
|
||||
KVConnector,
|
||||
get_kv_connector,
|
||||
)
|
||||
from vllm.v1.worker.gpu.lora_utils import LoraState
|
||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||
from vllm.v1.worker.gpu.mm.mrope_utils import MRopeState
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
@@ -168,6 +169,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
max_num_logits=self.max_num_reqs * (self.num_speculative_steps + 1),
|
||||
vocab_size=self.vocab_size,
|
||||
)
|
||||
# LoRA-related workers.
|
||||
self.lora_state = LoraState(max_num_reqs=self.max_num_reqs)
|
||||
|
||||
# Buffers for CPU-to-GPU copies.
|
||||
self.tmp_idx_mapping = UvaBufferPool(self.max_num_reqs, torch.int32)
|
||||
@@ -426,6 +429,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.supports_mm_inputs:
|
||||
self.encoder_runner.remove_request(req_id)
|
||||
self.prompt_logprobs_worker.remove_request(req_id)
|
||||
self.lora_state.remove_request(req_id)
|
||||
|
||||
def free_states(self, scheduler_output: SchedulerOutput) -> None:
|
||||
if self.supports_mm_inputs:
|
||||
@@ -444,7 +448,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
prompt_len=prompt_len,
|
||||
prefill_token_ids=new_req_data.prefill_token_ids,
|
||||
num_computed_tokens=new_req_data.num_computed_tokens,
|
||||
lora_request=new_req_data.lora_request,
|
||||
)
|
||||
req_index = self.req_states.req_id_to_index[req_id]
|
||||
|
||||
@@ -469,6 +472,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.prompt_logprobs_worker.add_request(
|
||||
req_id, req_index, new_req_data.sampling_params
|
||||
)
|
||||
self.lora_state.add_request(req_id, req_index, new_req_data.lora_request)
|
||||
|
||||
if scheduler_output.scheduled_new_reqs:
|
||||
self.req_states.apply_staged_writes()
|
||||
@@ -841,7 +845,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
if self.lora_config:
|
||||
# Activate LoRA adapters.
|
||||
lora_inputs = self.req_states.make_lora_inputs(
|
||||
lora_inputs = self.lora_state.make_lora_inputs(
|
||||
input_batch.req_ids,
|
||||
input_batch.idx_mapping_np,
|
||||
input_batch.num_scheduled_tokens,
|
||||
|
||||
Reference in New Issue
Block a user