[Model Runner V2][Bugfix] Fix MRV2 LoRA warmup (#35536)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-06-15 13:17:23 -07:00
committed by GitHub
co-authored by Nick Hill Woosuk Kwon
parent cd9078fe59
commit eacff17c8d
5 changed files with 227 additions and 63 deletions
+15 -3
View File
@@ -21,6 +21,7 @@ def sync_cudagraph_and_dp_padding(
uniform_token_count: int | None,
dp_size: int,
dp_rank: int,
num_active_loras: int = 0,
) -> tuple[BatchExecutionDescriptor, torch.Tensor | None]:
"""
Coordinates the batch descriptor and DP padding across all ranks.
@@ -53,6 +54,7 @@ def sync_cudagraph_and_dp_padding(
cg_mode=CUDAGraphMode.NONE,
num_tokens=num_tokens,
num_reqs=num_reqs,
num_active_loras=desired_batch_desc.num_active_loras,
), num_tokens_across_dp
assert cudagraph_manager is not None, (
@@ -68,9 +70,13 @@ def sync_cudagraph_and_dp_padding(
synced_uniform_token_count = None
# Dispatch for the final synced values, use num_reqs instead of synced_num_reqs
# so we don't perform request padding for PIECEWISE graphs
# so we don't perform request padding for PIECEWISE graphs.
# num_active_loras is per-rank and doesn't need cross-rank agreement.
synced_desc = cudagraph_manager.dispatch(
num_reqs, synced_num_tokens, synced_uniform_token_count
num_reqs,
synced_num_tokens,
synced_uniform_token_count,
num_active_loras=num_active_loras,
)
# Update num_tokens_across_dp to reflect padded size.
@@ -87,12 +93,14 @@ def dispatch_cg_and_sync_dp(
dp_size: int,
dp_rank: int,
need_eager: bool = False,
num_active_loras: int = 0,
) -> tuple[BatchExecutionDescriptor, torch.Tensor | None]:
if need_eager:
batch_desc = BatchExecutionDescriptor(
cg_mode=CUDAGraphMode.NONE,
num_tokens=num_tokens,
num_reqs=num_reqs,
num_active_loras=num_active_loras,
)
else:
assert cudagraph_manager is not None, (
@@ -100,7 +108,10 @@ def dispatch_cg_and_sync_dp(
"where need_eager must be True"
)
batch_desc = cudagraph_manager.dispatch(
num_reqs, num_tokens, uniform_token_count
num_reqs,
num_tokens,
uniform_token_count,
num_active_loras=num_active_loras,
)
if dp_size == 1:
@@ -114,4 +125,5 @@ def dispatch_cg_and_sync_dp(
uniform_token_count,
dp_size,
dp_rank,
num_active_loras=num_active_loras,
)