[Model Runner V2][Bugfix] Fix MRV2 LoRA warmup (#35536)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Nick Hill
Woosuk Kwon
parent
cd9078fe59
commit
eacff17c8d
@@ -37,7 +37,6 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.layers import LoRAMapping
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
initialize_mamba_ssu_backend,
|
||||
)
|
||||
@@ -88,7 +87,12 @@ from vllm.v1.worker.gpu.kv_connector import (
|
||||
KVConnector,
|
||||
get_kv_connector,
|
||||
)
|
||||
from vllm.v1.worker.gpu.lora_utils import LoraState
|
||||
from vllm.v1.worker.gpu.lora_utils import (
|
||||
LoraState,
|
||||
create_lora_capture_hook,
|
||||
get_lora_capture_cases,
|
||||
get_num_active_loras_for_dispatch,
|
||||
)
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.lora import set_active_mm_loras
|
||||
from vllm.v1.worker.gpu.model_states import init_model_state
|
||||
@@ -234,8 +238,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.prompt_logprobs_worker: PromptLogprobsWorker | None = None
|
||||
self.structured_outputs_worker: StructuredOutputsWorker | None = None
|
||||
self.cudagraph_manager: ModelCudaGraphManager | None = None
|
||||
|
||||
# LoRA-related workers.
|
||||
self.lora_state = LoraState(max_num_reqs=self.max_num_reqs)
|
||||
self.lora_capture_cases = [0]
|
||||
if self.lora_config:
|
||||
self.lora_capture_cases = get_lora_capture_cases(
|
||||
self.lora_config, self.compilation_config
|
||||
)
|
||||
|
||||
# KV Connector if configured.
|
||||
self.kv_connector: KVConnector = NO_OP_KV_CONNECTOR
|
||||
|
||||
@@ -458,6 +469,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.device,
|
||||
cudagraph_mode,
|
||||
decode_query_len=self.decode_query_len,
|
||||
lora_capture_cases=self.lora_capture_cases,
|
||||
)
|
||||
if self.speculator is not None:
|
||||
self.speculator.init_cudagraph_manager(cudagraph_mode)
|
||||
@@ -540,14 +552,22 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
assert self.intermediate_tensors is not None
|
||||
intermediate_tensors = self.intermediate_tensors[:num_tokens]
|
||||
|
||||
# Execute the model.
|
||||
self.execute_model(
|
||||
dummy_scheduler_output,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
dummy_run=True,
|
||||
skip_attn_for_dummy_run=skip_attn,
|
||||
is_profile=is_profile,
|
||||
)
|
||||
max_loras = self.lora_config.max_loras if self.lora_config is not None else 0
|
||||
with self.maybe_dummy_run_with_lora(
|
||||
self.lora_config,
|
||||
num_scheduled_tokens=np.array(num_tokens_per_request, dtype=np.int32),
|
||||
num_sampled_tokens=None,
|
||||
remove_lora=True,
|
||||
num_active_loras=max_loras,
|
||||
):
|
||||
# Execute the model.
|
||||
self.execute_model(
|
||||
dummy_scheduler_output,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
dummy_run=True,
|
||||
skip_attn_for_dummy_run=skip_attn,
|
||||
is_profile=is_profile,
|
||||
)
|
||||
self.kv_connector.set_disabled(False)
|
||||
|
||||
# Non-last PP ranks don't produce output for sampling.
|
||||
@@ -694,6 +714,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.kv_cache_config,
|
||||
has_lora=self.lora_config is not None,
|
||||
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
|
||||
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
|
||||
)
|
||||
if self.speculator is not None:
|
||||
self.speculator.capture(attn_states)
|
||||
@@ -1105,6 +1126,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
max_query_len = max(scheduler_output.num_scheduled_tokens.values())
|
||||
uniform_tok_count = get_uniform_token_count(num_reqs, num_toks, max_query_len)
|
||||
|
||||
num_active_loras = 0
|
||||
if self.lora_config:
|
||||
req_ids = list(scheduler_output.num_scheduled_tokens.keys())
|
||||
num_active_loras = get_num_active_loras_for_dispatch(
|
||||
self.lora_config, self.lora_state, req_ids, dummy_run
|
||||
)
|
||||
|
||||
skip_compiled = False
|
||||
if self.is_encoder_decoder and scheduler_output.scheduled_encoder_inputs:
|
||||
# Encoder-decoder models such as Whisper should run eager/non-compiled
|
||||
@@ -1120,6 +1148,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.dp_size,
|
||||
self.dp_rank,
|
||||
need_eager=is_profile or skip_compiled,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
|
||||
if batch_desc.num_tokens == 0:
|
||||
@@ -1157,31 +1186,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
block_tables = None
|
||||
slot_mappings = None
|
||||
if self.lora_config:
|
||||
# program a no-LoRA mapping here so kernels early-exit instead of
|
||||
# reading uninitialized metadata during dummy runs.
|
||||
# FIXME: Replace this with LoRA warmup:
|
||||
# https://github.com/vllm-project/vllm/pull/35536
|
||||
assert hasattr(self, "lora_manager")
|
||||
adapter_manager = self.lora_manager._adapter_manager
|
||||
adapter_manager.set_adapter_mapping(
|
||||
LoRAMapping(
|
||||
index_mapping=(0,) * input_batch.num_tokens_after_padding,
|
||||
prompt_mapping=(0,) * input_batch.num_reqs,
|
||||
is_prefill=True,
|
||||
)
|
||||
)
|
||||
seen_wrappers: set[int] = set()
|
||||
for punica_wrapper in adapter_manager.punica_wrapper_mapping.values():
|
||||
if id(punica_wrapper) in seen_wrappers:
|
||||
continue
|
||||
seen_wrappers.add(id(punica_wrapper))
|
||||
for kernel_meta in (
|
||||
punica_wrapper.token_mapping_meta, # type: ignore[attr-defined]
|
||||
punica_wrapper.prompt_mapping_meta, # type: ignore[attr-defined]
|
||||
):
|
||||
kernel_meta.no_lora_flag_cpu[0] = False
|
||||
kernel_meta.num_active_loras_cpu[0] = 1
|
||||
|
||||
attn_metadata = None
|
||||
slot_mappings_by_layer = None
|
||||
@@ -1258,6 +1262,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
batch_descriptor = BatchDescriptor(
|
||||
num_tokens=input_batch.num_tokens_after_padding,
|
||||
has_lora=self.lora_config is not None,
|
||||
num_active_loras=batch_desc.num_active_loras,
|
||||
)
|
||||
|
||||
with set_forward_context(
|
||||
|
||||
Reference in New Issue
Block a user