[Model Runner V2][Bugfix] Fix MRV2 LoRA warmup (#35536)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Nick Hill
Woosuk Kwon
parent
cd9078fe59
commit
eacff17c8d
@@ -21,6 +21,7 @@ def sync_cudagraph_and_dp_padding(
|
||||
uniform_token_count: int | None,
|
||||
dp_size: int,
|
||||
dp_rank: int,
|
||||
num_active_loras: int = 0,
|
||||
) -> tuple[BatchExecutionDescriptor, torch.Tensor | None]:
|
||||
"""
|
||||
Coordinates the batch descriptor and DP padding across all ranks.
|
||||
@@ -53,6 +54,7 @@ def sync_cudagraph_and_dp_padding(
|
||||
cg_mode=CUDAGraphMode.NONE,
|
||||
num_tokens=num_tokens,
|
||||
num_reqs=num_reqs,
|
||||
num_active_loras=desired_batch_desc.num_active_loras,
|
||||
), num_tokens_across_dp
|
||||
|
||||
assert cudagraph_manager is not None, (
|
||||
@@ -68,9 +70,13 @@ def sync_cudagraph_and_dp_padding(
|
||||
synced_uniform_token_count = None
|
||||
|
||||
# Dispatch for the final synced values, use num_reqs instead of synced_num_reqs
|
||||
# so we don't perform request padding for PIECEWISE graphs
|
||||
# so we don't perform request padding for PIECEWISE graphs.
|
||||
# num_active_loras is per-rank and doesn't need cross-rank agreement.
|
||||
synced_desc = cudagraph_manager.dispatch(
|
||||
num_reqs, synced_num_tokens, synced_uniform_token_count
|
||||
num_reqs,
|
||||
synced_num_tokens,
|
||||
synced_uniform_token_count,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
|
||||
# Update num_tokens_across_dp to reflect padded size.
|
||||
@@ -87,12 +93,14 @@ def dispatch_cg_and_sync_dp(
|
||||
dp_size: int,
|
||||
dp_rank: int,
|
||||
need_eager: bool = False,
|
||||
num_active_loras: int = 0,
|
||||
) -> tuple[BatchExecutionDescriptor, torch.Tensor | None]:
|
||||
if need_eager:
|
||||
batch_desc = BatchExecutionDescriptor(
|
||||
cg_mode=CUDAGraphMode.NONE,
|
||||
num_tokens=num_tokens,
|
||||
num_reqs=num_reqs,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
else:
|
||||
assert cudagraph_manager is not None, (
|
||||
@@ -100,7 +108,10 @@ def dispatch_cg_and_sync_dp(
|
||||
"where need_eager must be True"
|
||||
)
|
||||
batch_desc = cudagraph_manager.dispatch(
|
||||
num_reqs, num_tokens, uniform_token_count
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
uniform_token_count,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
|
||||
if dp_size == 1:
|
||||
@@ -114,4 +125,5 @@ def dispatch_cg_and_sync_dp(
|
||||
uniform_token_count,
|
||||
dp_size,
|
||||
dp_rank,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user