[Core] Refactor padding logic and pad for CUDA graphs before attention metadata building (#28579)
This commit is contained in:
@@ -9,6 +9,7 @@ from vllm.config import ParallelConfig
|
||||
from vllm.distributed.parallel_state import get_dp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.worker.ubatch_utils import (
|
||||
UBatchSlice,
|
||||
UBatchSlices,
|
||||
check_ubatch_thresholds,
|
||||
create_ubatch_slices,
|
||||
@@ -88,6 +89,17 @@ def _post_process_dp_padding(tensor: torch.Tensor, should_dp_pad: bool) -> torch
|
||||
return num_tokens_across_dp.cpu()
|
||||
|
||||
|
||||
# This just pads the second ubatch slice out to the total number of tokens
|
||||
# (num_tokens + padding) since we do `create_ubatch_slices` before applying DP padding.
|
||||
def _pad_out_ubatch_slice(ubatch_slices: UBatchSlices, num_total_tokens: int):
|
||||
padded_second_ubatch_slice = slice(
|
||||
ubatch_slices[1].token_slice.start, num_total_tokens
|
||||
)
|
||||
ubatch_slices[1] = UBatchSlice(
|
||||
padded_second_ubatch_slice, padded_second_ubatch_slice
|
||||
)
|
||||
|
||||
|
||||
def _synchronize_dp_ranks(
|
||||
num_tokens_unpadded: int,
|
||||
num_tokens_padded: int,
|
||||
@@ -220,11 +232,14 @@ def coordinate_batch_across_dp(
|
||||
# to the second ubatch in pad_out_ubatch_slice after attention
|
||||
# metadata creation
|
||||
assert num_tokens_after_padding is not None
|
||||
token_split_point = int(num_tokens_after_padding[0].item()) // 2
|
||||
num_tokens_padded = int(num_tokens_after_padding[0].item())
|
||||
token_split_point = int(num_tokens_padded) // 2
|
||||
|
||||
assert num_scheduled_tokens_per_request is not None
|
||||
ubatch_slices = create_ubatch_slices(
|
||||
num_scheduled_tokens_per_request, token_split_point
|
||||
)
|
||||
ubatch_slices = _pad_out_ubatch_slice(ubatch_slices, num_tokens_padded)
|
||||
assert sum(s.num_tokens for s in ubatch_slices) == num_tokens_padded
|
||||
|
||||
return (ubatch_slices, num_tokens_after_padding)
|
||||
|
||||
Reference in New Issue
Block a user