[Perf][Attention] Avoid CPU-GPU sync in DCP sequence lengths (#58169)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
96e203b08e
commit
6632ed559e
@@ -1096,6 +1096,7 @@ def get_dcp_local_seq_lens(
|
|||||||
Only consider dcp now, we can extend the case of cp based on this.
|
Only consider dcp now, we can extend the case of cp based on this.
|
||||||
"""
|
"""
|
||||||
seq_lens_i32 = seq_lens.to(torch.int32)
|
seq_lens_i32 = seq_lens.to(torch.int32)
|
||||||
|
rank_offsets: int | torch.Tensor
|
||||||
if dcp_rank is None:
|
if dcp_rank is None:
|
||||||
rank_offsets = torch.arange(
|
rank_offsets = torch.arange(
|
||||||
dcp_size,
|
dcp_size,
|
||||||
@@ -1113,7 +1114,8 @@ def get_dcp_local_seq_lens(
|
|||||||
rank_offsets = dcp_rank
|
rank_offsets = dcp_rank
|
||||||
seq_lens_tiled = seq_lens_i32
|
seq_lens_tiled = seq_lens_i32
|
||||||
else:
|
else:
|
||||||
rank_offsets = torch.tensor(dcp_rank, dtype=torch.int32, device=seq_lens.device)
|
# Use the Python scalar directly to avoid a synchronizing H2D copy.
|
||||||
|
rank_offsets = dcp_rank
|
||||||
seq_lens_tiled = seq_lens_i32
|
seq_lens_tiled = seq_lens_i32
|
||||||
base = (
|
base = (
|
||||||
seq_lens_tiled
|
seq_lens_tiled
|
||||||
|
|||||||
Reference in New Issue
Block a user