[ROCm][Perf] Enable DPA+ETP prefill compute/comm dual-batch-overlap for DeepSeek-V4 (#57773)
Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com> Signed-off-by: shen-shanshan <467638484@qq.com>
This commit is contained in:
@@ -213,6 +213,7 @@ class UBatchWrapper:
|
||||
def _run_ubatches(self, ubatch_metadata, model) -> torch.Tensor:
|
||||
@torch.inference_mode()
|
||||
def _ubatch_thread(results, model, ubatch_metadata):
|
||||
torch.accelerator.set_device_index(self.device)
|
||||
with ubatch_metadata.context:
|
||||
model_output = model(
|
||||
input_ids=ubatch_metadata.input_ids,
|
||||
@@ -291,6 +292,7 @@ class UBatchWrapper:
|
||||
compute_stream=compute_stream,
|
||||
forward_contexts=forward_contexts,
|
||||
ready_barrier=self.ready_barrier,
|
||||
token_offsets=[s.token_slice.start for s in ubatch_slices],
|
||||
)
|
||||
|
||||
ubatch_metadata: list[UbatchMetadata] = []
|
||||
|
||||
Reference in New Issue
Block a user