[ROCm][Perf] Enable DPA+ETP prefill compute/comm dual-batch-overlap for DeepSeek-V4 (#57773)

Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
Signed-off-by: shen-shanshan <467638484@qq.com>
This commit is contained in:
Shanshan Shen
2026-10-15 11:16:48 +08:00
committed by GitHub
parent 5d5c3d7c43
commit f46021a7fa
9 changed files with 319 additions and 28 deletions
+2
View File
@@ -213,6 +213,7 @@ class UBatchWrapper:
def _run_ubatches(self, ubatch_metadata, model) -> torch.Tensor:
@torch.inference_mode()
def _ubatch_thread(results, model, ubatch_metadata):
torch.accelerator.set_device_index(self.device)
with ubatch_metadata.context:
model_output = model(
input_ids=ubatch_metadata.input_ids,
@@ -291,6 +292,7 @@ class UBatchWrapper:
compute_stream=compute_stream,
forward_contexts=forward_contexts,
ready_barrier=self.ready_barrier,
token_offsets=[s.token_slice.start for s in ubatch_slices],
)
ubatch_metadata: list[UbatchMetadata] = []