[Perf][Attention] Avoid CPU-GPU sync in DCP sequence lengths (#58169)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
96e203b08e
commit
6632ed559e
@@ -1096,6 +1096,7 @@ def get_dcp_local_seq_lens(
|
||||
Only consider dcp now, we can extend the case of cp based on this.
|
||||
"""
|
||||
seq_lens_i32 = seq_lens.to(torch.int32)
|
||||
rank_offsets: int | torch.Tensor
|
||||
if dcp_rank is None:
|
||||
rank_offsets = torch.arange(
|
||||
dcp_size,
|
||||
@@ -1113,7 +1114,8 @@ def get_dcp_local_seq_lens(
|
||||
rank_offsets = dcp_rank
|
||||
seq_lens_tiled = seq_lens_i32
|
||||
else:
|
||||
rank_offsets = torch.tensor(dcp_rank, dtype=torch.int32, device=seq_lens.device)
|
||||
# Use the Python scalar directly to avoid a synchronizing H2D copy.
|
||||
rank_offsets = dcp_rank
|
||||
seq_lens_tiled = seq_lens_i32
|
||||
base = (
|
||||
seq_lens_tiled
|
||||
|
||||
Reference in New Issue
Block a user