Reduce the kernel overhead when num of active loras is smaller than max loras. Multiple cuda graphs are captured for each num of active-loras. (#32005)
Signed-off-by: Yu Gong <yu3.gong@gmail.com>
This commit is contained in:
@@ -3082,6 +3082,7 @@ class GPUModelRunner(
|
||||
# be improved in model runner v2)
|
||||
force_uniform_decode: bool | None = None,
|
||||
force_has_lora: bool | None = None,
|
||||
force_num_active_loras: int | None = None,
|
||||
num_encoder_reqs: int = 0,
|
||||
) -> tuple[
|
||||
CUDAGraphMode,
|
||||
@@ -3103,11 +3104,13 @@ class GPUModelRunner(
|
||||
self.model_config.is_encoder_decoder and num_encoder_reqs > 0
|
||||
)
|
||||
|
||||
has_lora = (
|
||||
len(self.input_batch.lora_id_to_lora_request) > 0
|
||||
if force_has_lora is None
|
||||
else force_has_lora
|
||||
# Compute LoRA state for cudagraph dispatch
|
||||
num_active_loras = (
|
||||
force_num_active_loras
|
||||
if force_num_active_loras is not None
|
||||
else len(self.input_batch.lora_id_to_lora_request)
|
||||
)
|
||||
has_lora = num_active_loras > 0 if force_has_lora is None else force_has_lora
|
||||
|
||||
num_tokens_padded = self._pad_for_sequence_parallelism(num_tokens)
|
||||
dispatch_cudagraph = (
|
||||
@@ -3116,6 +3119,7 @@ class GPUModelRunner(
|
||||
has_lora=has_lora,
|
||||
uniform_decode=uniform_decode,
|
||||
disable_full=disable_full,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
if not force_eager
|
||||
else (CUDAGraphMode.NONE, BatchDescriptor(num_tokens_padded))
|
||||
@@ -4606,8 +4610,8 @@ class GPUModelRunner(
|
||||
is_profile: bool = False,
|
||||
create_mixed_batch: bool = False,
|
||||
remove_lora: bool = True,
|
||||
activate_lora: bool = False,
|
||||
is_graph_capturing: bool = False,
|
||||
num_active_loras: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Run a dummy forward pass to warm up/profile run or capture the
|
||||
@@ -4630,7 +4634,8 @@ class GPUModelRunner(
|
||||
create_mixed_batch: If True, create a mixed batch with both decode
|
||||
(1 token) and prefill (multiple tokens) requests.
|
||||
remove_lora: If False, dummy LoRAs are not destroyed after the run
|
||||
activate_lora: If False, dummy_run is performed without LoRAs.
|
||||
num_active_loras: Number of distinct active LoRAs to capture for.
|
||||
LoRA is activated when num_active_loras > 0.
|
||||
"""
|
||||
mm_config = self.vllm_config.model_config.multimodal_config
|
||||
if mm_config and mm_config.mm_encoder_only:
|
||||
@@ -4712,7 +4717,10 @@ class GPUModelRunner(
|
||||
# `force_has_lora` is used for cudagraph capture; because LoRA is
|
||||
# activated later in the context manager, but we need to know the
|
||||
# LoRA state when determining the batch descriptor for capture
|
||||
force_has_lora=activate_lora,
|
||||
force_has_lora=num_active_loras > 0,
|
||||
# `force_num_active_loras` is used for cudagraph capture; because we
|
||||
# need to capture graphs for specific num_active_loras counts
|
||||
force_num_active_loras=num_active_loras,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -4782,8 +4790,8 @@ class GPUModelRunner(
|
||||
self.lora_config,
|
||||
num_scheduled_tokens,
|
||||
num_sampled_tokens,
|
||||
activate_lora,
|
||||
remove_lora,
|
||||
num_active_loras,
|
||||
):
|
||||
# Make sure padding doesn't exceed max_num_tokens
|
||||
assert num_tokens_padded <= self.max_num_tokens
|
||||
@@ -4884,7 +4892,10 @@ class GPUModelRunner(
|
||||
# lora cases when cudagraph_specialize_lora is enabled. This is a
|
||||
# short term mitigation for issue mentioned in
|
||||
# https://github.com/vllm-project/vllm/issues/28334
|
||||
if self.compilation_config.cudagraph_specialize_lora and activate_lora:
|
||||
if (
|
||||
self.compilation_config.cudagraph_specialize_lora
|
||||
and num_active_loras > 0
|
||||
):
|
||||
use_cudagraphs = False
|
||||
|
||||
self.drafter.dummy_run(
|
||||
@@ -5259,7 +5270,7 @@ class GPUModelRunner(
|
||||
# We skip EPLB here since we don't want to record dummy metrics
|
||||
for batch_desc in batch_descriptors:
|
||||
num_tokens = batch_desc.num_tokens
|
||||
activate_lora = batch_desc.has_lora
|
||||
num_active_loras = batch_desc.num_active_loras
|
||||
|
||||
# We currently only capture ubatched graphs when its a FULL
|
||||
# cudagraph, a uniform decode batch, and the number of tokens
|
||||
@@ -5286,7 +5297,7 @@ class GPUModelRunner(
|
||||
num_tokens,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
allow_microbatching=allow_microbatching,
|
||||
activate_lora=activate_lora,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
|
||||
# Capture run
|
||||
@@ -5294,7 +5305,7 @@ class GPUModelRunner(
|
||||
num_tokens,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
allow_microbatching=allow_microbatching,
|
||||
activate_lora=activate_lora,
|
||||
num_active_loras=num_active_loras,
|
||||
is_graph_capturing=True,
|
||||
)
|
||||
self.maybe_remove_all_loras(self.lora_config)
|
||||
|
||||
Reference in New Issue
Block a user