[feature] extend DBO to XBO (#30120)
Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com> Co-authored-by: root <root@hk01dgx028.cm.cluster>
This commit is contained in:
@@ -103,8 +103,10 @@ class UBatchWrapper:
|
||||
self.vllm_config = vllm_config
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
self.comm_stream = torch.cuda.Stream(device=device)
|
||||
# Two ubatch threads plus the main thread
|
||||
self.ready_barrier = threading.Barrier(3)
|
||||
# Ubatch threads plus the main thread
|
||||
self.ready_barrier = threading.Barrier(
|
||||
self.vllm_config.parallel_config.num_ubatches + 1
|
||||
)
|
||||
|
||||
self.cudagraphs: dict[int, CUDAGraphMetaData] = {}
|
||||
|
||||
@@ -309,7 +311,7 @@ class UBatchWrapper:
|
||||
create_forward_context(
|
||||
attn_metadata[i] if attn_metadata is not None else None,
|
||||
self.vllm_config,
|
||||
dp_metadata=dp_metadata,
|
||||
dp_metadata=dp_metadata[i],
|
||||
batch_descriptor=batch_descriptor,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
)
|
||||
@@ -417,18 +419,19 @@ class UBatchWrapper:
|
||||
|
||||
# We shouldn't be here unless we are running with multiple DP ranks
|
||||
assert dp_metadata is not None
|
||||
num_tokens_per_ubatch = (
|
||||
ubatch_slices[0].token_slice.stop - ubatch_slices[0].token_slice.start
|
||||
)
|
||||
dp_size = self.vllm_config.parallel_config.data_parallel_size
|
||||
ubatch_num_tokens_across_dp = torch.tensor(
|
||||
[num_tokens_per_ubatch] * dp_size, device="cpu", dtype=torch.int32
|
||||
)
|
||||
ubatch_dp_metadata = DPMetadata.make(
|
||||
self.vllm_config.parallel_config,
|
||||
num_tokens_per_ubatch,
|
||||
ubatch_num_tokens_across_dp,
|
||||
)
|
||||
ubatch_dp_metadata = []
|
||||
for ubatch_slice in ubatch_slices:
|
||||
dp_size = self.vllm_config.parallel_config.data_parallel_size
|
||||
ubatch_num_tokens_across_dp = torch.tensor(
|
||||
[ubatch_slice.num_tokens] * dp_size, device="cpu", dtype=torch.int32
|
||||
)
|
||||
ubatch_dp_metadata.append(
|
||||
DPMetadata.make(
|
||||
self.vllm_config.parallel_config,
|
||||
ubatch_slice.num_tokens,
|
||||
ubatch_num_tokens_across_dp,
|
||||
)
|
||||
)
|
||||
|
||||
if (
|
||||
num_tokens not in self.cudagraphs
|
||||
@@ -464,7 +467,7 @@ class UBatchWrapper:
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
compute_stream=compute_stream,
|
||||
dp_metadata=dp_metadata,
|
||||
dp_metadata=ubatch_dp_metadata,
|
||||
batch_descriptor=batch_descriptor,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user