[Model Runner V2][Bugfix] Fix MRV2 LoRA warmup (#35536)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Nick Hill
Woosuk Kwon
parent
cd9078fe59
commit
eacff17c8d
@@ -3,6 +3,7 @@
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from itertools import product
|
||||
from typing import Any, NamedTuple, Protocol
|
||||
|
||||
import torch
|
||||
@@ -56,6 +57,7 @@ class BatchExecutionDescriptor:
|
||||
num_tokens: int
|
||||
num_reqs: int | None # None means no request padding is needed (PIECEWISE graphs)
|
||||
uniform_token_count: int | None = None
|
||||
num_active_loras: int = 0
|
||||
|
||||
|
||||
class CreateForwardFn(Protocol):
|
||||
@@ -75,6 +77,7 @@ def _is_compatible(
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
uniform_token_count: int | None,
|
||||
num_active_loras: int,
|
||||
) -> bool:
|
||||
# desc.uniform_token_count=None (PIECEWISE) can handle any uniform_token_count
|
||||
# desc.num_reqs=None means no request padding needed (PIECEWISE)
|
||||
@@ -85,6 +88,7 @@ def _is_compatible(
|
||||
)
|
||||
and (desc.num_reqs is None or desc.num_reqs >= num_reqs)
|
||||
and desc.num_tokens >= num_tokens
|
||||
and desc.num_active_loras == num_active_loras
|
||||
)
|
||||
|
||||
|
||||
@@ -111,6 +115,7 @@ class CudaGraphManager:
|
||||
device: torch.device,
|
||||
cudagraph_mode: CUDAGraphMode,
|
||||
decode_query_len: int,
|
||||
lora_capture_cases: list[int] | None = None,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
@@ -124,12 +129,17 @@ class CudaGraphManager:
|
||||
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
self.is_first_pp_rank = get_pp_group().is_first_rank
|
||||
self.is_last_pp_rank = get_pp_group().is_last_rank
|
||||
self.lora_capture_cases = lora_capture_cases or [0]
|
||||
# Precompute actual num_active_loras -> captured case mapping so that
|
||||
# dispatch() is a plain dict lookup instead of a per-call bisect.
|
||||
self._lora_dispatch_map, self._max_lora_case = self._build_lora_dispatch_map()
|
||||
|
||||
self.graphs: dict[BatchExecutionDescriptor, torch.cuda.CUDAGraph] = {}
|
||||
self.pool = current_platform.get_global_graph_pool() if cudagraph_mode else None
|
||||
|
||||
self._graphs_captured = False
|
||||
self._candidates: list[list[BatchExecutionDescriptor]] = []
|
||||
|
||||
self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {}
|
||||
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
|
||||
# adjust the cudagraph sizes to be a multiple of the uniform decode query length
|
||||
self.compilation_config.adjust_cudagraph_sizes_for_spec_decode(
|
||||
@@ -144,6 +154,32 @@ class CudaGraphManager:
|
||||
)
|
||||
self.breakable_cg_runner: BreakableCUDAGraphWrapper | None = None
|
||||
|
||||
def _build_lora_dispatch_map(self) -> tuple[dict[int, int], int]:
|
||||
"""Precompute actual num_active_loras -> effective captured case.
|
||||
|
||||
Mirrors the num_tokens candidate expansion in ``_init_candidates``:
|
||||
every possible active-LoRA count is mapped ahead of time to the
|
||||
smallest captured case that can serve it, so ``dispatch`` is a plain
|
||||
dict lookup instead of a per-call bisect.
|
||||
"""
|
||||
captured_with_lora = sorted(c for c in self.lora_capture_cases if c > 0)
|
||||
if not captured_with_lora:
|
||||
return {}, 0
|
||||
dispatch_map: dict[int, int] = {}
|
||||
case_idx = 0
|
||||
for n in range(1, captured_with_lora[-1] + 1):
|
||||
while captured_with_lora[case_idx] < n:
|
||||
case_idx += 1
|
||||
dispatch_map[n] = captured_with_lora[case_idx]
|
||||
return dispatch_map, captured_with_lora[-1]
|
||||
|
||||
def _resolve_effective_loras(self, num_active_loras: int) -> int:
|
||||
"""Map an actual active-LoRA count to its captured graph case."""
|
||||
if num_active_loras <= 0 or not self._lora_dispatch_map:
|
||||
return num_active_loras
|
||||
# Counts above the largest captured case clamp to it.
|
||||
return self._lora_dispatch_map.get(num_active_loras, self._max_lora_case)
|
||||
|
||||
def _init_candidates(self) -> None:
|
||||
"""Build priority-ordered candidate lists for each token count."""
|
||||
capture_sizes = self.compilation_config.cudagraph_capture_sizes
|
||||
@@ -156,10 +192,14 @@ class CudaGraphManager:
|
||||
mixed_mode = self.cudagraph_mode.mixed_mode()
|
||||
separate_decode_routine = self.cudagraph_mode.separate_routine()
|
||||
|
||||
descs_by_token_count = defaultdict(list)
|
||||
descs_by_token_lora: dict[tuple[int, int], list[BatchExecutionDescriptor]] = (
|
||||
defaultdict(list)
|
||||
)
|
||||
descs_by_mode = defaultdict(list)
|
||||
|
||||
for num_tokens in capture_sizes:
|
||||
for num_tokens, num_active_loras in product(
|
||||
capture_sizes, self.lora_capture_cases
|
||||
):
|
||||
# Capture uniform decode specfifc graphs if required
|
||||
# (i.e. separate decode routine)
|
||||
if (
|
||||
@@ -172,9 +212,10 @@ class CudaGraphManager:
|
||||
num_tokens=num_tokens,
|
||||
num_reqs=num_tokens // self.decode_query_len,
|
||||
uniform_token_count=self.decode_query_len,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
descs_by_mode[decode_mode].append(desc)
|
||||
descs_by_token_count[num_tokens].append(desc)
|
||||
descs_by_token_lora[(num_tokens, num_active_loras)].append(desc)
|
||||
|
||||
if mixed_mode:
|
||||
# for PIECEWISE graphs there is no limit on requests when replaying
|
||||
@@ -189,21 +230,25 @@ class CudaGraphManager:
|
||||
cg_mode=mixed_mode,
|
||||
num_tokens=num_tokens,
|
||||
num_reqs=num_reqs,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
descs_by_mode[mixed_mode].append(desc)
|
||||
descs_by_token_count[num_tokens].append(desc)
|
||||
descs_by_token_lora[(num_tokens, num_active_loras)].append(desc)
|
||||
|
||||
if not descs_by_token_count:
|
||||
if not descs_by_token_lora:
|
||||
return
|
||||
|
||||
sorted_padded = sorted(descs_by_token_count.keys())
|
||||
self._candidates = [[] for _ in range(sorted_padded[-1] + 1)]
|
||||
|
||||
all_token_counts = sorted({k[0] for k in descs_by_token_lora})
|
||||
current_range_start = 0
|
||||
for cg_size in sorted_padded:
|
||||
for i in range(current_range_start, cg_size + 1):
|
||||
self._candidates[i] = descs_by_token_count[cg_size]
|
||||
current_range_start = cg_size + 1
|
||||
for token_cg_size in all_token_counts:
|
||||
for i in range(current_range_start, token_cg_size + 1):
|
||||
for num_active_loras in self.lora_capture_cases:
|
||||
staging_key = (token_cg_size, num_active_loras)
|
||||
if staging_key in descs_by_token_lora:
|
||||
self._candidates[(i, num_active_loras)] = descs_by_token_lora[
|
||||
staging_key
|
||||
]
|
||||
current_range_start = token_cg_size + 1
|
||||
|
||||
for mode, descs in descs_by_mode.items():
|
||||
descs.sort(key=lambda d: d.num_tokens, reverse=True)
|
||||
@@ -289,14 +334,27 @@ class CudaGraphManager:
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
uniform_token_count: int | None,
|
||||
num_active_loras: int,
|
||||
) -> BatchExecutionDescriptor:
|
||||
"""Find matching cudagraph descriptor from priority-ordered candidates."""
|
||||
if self._graphs_captured and 0 < num_tokens < len(self._candidates):
|
||||
for desc in self._candidates[num_tokens]:
|
||||
if _is_compatible(desc, num_reqs, num_tokens, uniform_token_count):
|
||||
|
||||
effective_loras = self._resolve_effective_loras(num_active_loras)
|
||||
key = (num_tokens, effective_loras)
|
||||
if self._graphs_captured and num_tokens > 0 and key in self._candidates:
|
||||
for desc in self._candidates[key]:
|
||||
if _is_compatible(
|
||||
desc,
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
uniform_token_count,
|
||||
effective_loras,
|
||||
):
|
||||
return desc
|
||||
return BatchExecutionDescriptor(
|
||||
cg_mode=CUDAGraphMode.NONE, num_tokens=num_tokens, num_reqs=num_reqs
|
||||
cg_mode=CUDAGraphMode.NONE,
|
||||
num_tokens=num_tokens,
|
||||
num_reqs=num_reqs,
|
||||
num_active_loras=effective_loras,
|
||||
)
|
||||
|
||||
def run_fullgraph(self, desc: BatchExecutionDescriptor):
|
||||
@@ -337,9 +395,15 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
device: torch.device,
|
||||
cudagraph_mode: CUDAGraphMode,
|
||||
decode_query_len: int,
|
||||
lora_capture_cases: list[int] | None = None,
|
||||
):
|
||||
super().__init__(vllm_config, device, cudagraph_mode, decode_query_len)
|
||||
# Used for FULL CUDA graphs. PW CUDA graphs do not use these.
|
||||
super().__init__(
|
||||
vllm_config,
|
||||
device,
|
||||
cudagraph_mode,
|
||||
decode_query_len,
|
||||
lora_capture_cases=lora_capture_cases,
|
||||
)
|
||||
self.hidden_states: torch.Tensor | None = None
|
||||
self.aux_hidden_states: list[torch.Tensor] = []
|
||||
self.use_aux_hidden_state_outputs = False
|
||||
@@ -356,6 +420,7 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
kv_cache_config: KVCacheConfig,
|
||||
has_lora: bool = False,
|
||||
use_aux_hidden_state_outputs: bool = False,
|
||||
lora_capture_hook: Callable[[int, int, int], None] | None = None,
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> dict[BatchExecutionDescriptor, AttentionStatePair]:
|
||||
"""Capture CUDA graphs for model forward pass."""
|
||||
@@ -372,6 +437,11 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
]:
|
||||
num_tokens = desc.num_tokens
|
||||
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
|
||||
|
||||
# Set LoRA state before capture so kernels see correct adapters.
|
||||
if lora_capture_hook is not None:
|
||||
lora_capture_hook(desc.num_active_loras, num_reqs, num_tokens)
|
||||
|
||||
num_tokens_across_dp = (
|
||||
torch.full((self.dp_size,), num_tokens, dtype=torch.int32, device="cpu")
|
||||
if self.dp_size > 1
|
||||
@@ -406,7 +476,9 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
if cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
assert attn_metadata is None
|
||||
batch_descriptor = BatchDescriptor(
|
||||
num_tokens=num_tokens, has_lora=has_lora
|
||||
num_tokens=num_tokens,
|
||||
has_lora=has_lora,
|
||||
num_active_loras=desc.num_active_loras,
|
||||
)
|
||||
with set_forward_context(
|
||||
attn_metadata,
|
||||
|
||||
Reference in New Issue
Block a user