[feature] extend DBO to XBO (#30120)
Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com> Co-authored-by: root <root@hk01dgx028.cm.cluster>
This commit is contained in:
@@ -2987,7 +2987,7 @@ class GPUModelRunner(
|
||||
|
||||
cascade_attn_prefix_lens = None
|
||||
# Disable cascade attention when using microbatching (DBO)
|
||||
if self.cascade_attn_enabled and not self.parallel_config.enable_dbo:
|
||||
if self.cascade_attn_enabled and not self.parallel_config.use_ubatching:
|
||||
# Pre-compute cascade attention prefix lengths
|
||||
cascade_attn_prefix_lens = self._compute_cascade_attn_prefix_lens(
|
||||
num_scheduled_tokens_np,
|
||||
@@ -3028,6 +3028,13 @@ class GPUModelRunner(
|
||||
num_scheduled_tokens_np,
|
||||
num_tokens_padded,
|
||||
num_reqs_padded,
|
||||
self.parallel_config.num_ubatches,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"ubatch_slices: %s, ubatch_slices_padded: %s",
|
||||
ubatch_slices,
|
||||
ubatch_slices_padded,
|
||||
)
|
||||
|
||||
pad_attn = cudagraph_mode == CUDAGraphMode.FULL
|
||||
@@ -3710,11 +3717,14 @@ class GPUModelRunner(
|
||||
# wrap the model with full cudagraph wrapper if needed.
|
||||
cudagraph_mode = self.compilation_config.cudagraph_mode
|
||||
assert cudagraph_mode is not None
|
||||
if cudagraph_mode.has_full_cudagraphs() and not self.parallel_config.enable_dbo:
|
||||
if (
|
||||
cudagraph_mode.has_full_cudagraphs()
|
||||
and not self.parallel_config.use_ubatching
|
||||
):
|
||||
self.model = CUDAGraphWrapper(
|
||||
self.model, self.vllm_config, runtime_mode=CUDAGraphMode.FULL
|
||||
)
|
||||
elif self.parallel_config.enable_dbo:
|
||||
elif self.parallel_config.use_ubatching:
|
||||
if cudagraph_mode.has_full_cudagraphs():
|
||||
self.model = UBatchWrapper(
|
||||
self.model, self.vllm_config, CUDAGraphMode.FULL, self.device
|
||||
@@ -4095,7 +4105,16 @@ class GPUModelRunner(
|
||||
batch_desc.num_reqs if batch_desc.num_reqs is not None else num_reqs
|
||||
)
|
||||
ubatch_slices, ubatch_slices_padded = maybe_create_ubatch_slices(
|
||||
should_ubatch, num_scheduled_tokens, num_tokens_padded, num_reqs_padded
|
||||
should_ubatch,
|
||||
num_scheduled_tokens,
|
||||
num_tokens_padded,
|
||||
num_reqs_padded,
|
||||
self.vllm_config.parallel_config.num_ubatches,
|
||||
)
|
||||
logger.debug(
|
||||
"ubatch_slices: %s, ubatch_slices_padded: %s",
|
||||
ubatch_slices,
|
||||
ubatch_slices_padded,
|
||||
)
|
||||
|
||||
attn_metadata: PerLayerAttnMetadata | None = None
|
||||
@@ -4644,7 +4663,7 @@ class GPUModelRunner(
|
||||
# is above the threshold. Otherwise we just capture a non-ubatched
|
||||
# version of the graph
|
||||
allow_microbatching = (
|
||||
self.parallel_config.enable_dbo
|
||||
self.parallel_config.use_ubatching
|
||||
and cudagraph_runtime_mode == CUDAGraphMode.FULL
|
||||
and uniform_decode
|
||||
and check_ubatch_thresholds(
|
||||
@@ -4779,8 +4798,8 @@ class GPUModelRunner(
|
||||
if kv_cache_group_id < len(kernel_block_sizes)
|
||||
else None,
|
||||
num_metadata_builders=1
|
||||
if not self.parallel_config.enable_dbo
|
||||
else 2,
|
||||
if not self.parallel_config.use_ubatching
|
||||
else self.parallel_config.num_ubatches,
|
||||
)
|
||||
# Calculate reorder batch threshold (if needed)
|
||||
# Note (tdoublep): do this *after* constructing builders,
|
||||
|
||||
Reference in New Issue
Block a user