[Core] Avoid seq_lens_cpu GPU->CPU sync (#40654)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -60,6 +60,8 @@ class InputBatch:
|
||||
query_start_loc_np: np.ndarray
|
||||
# [num_reqs]
|
||||
seq_lens: torch.Tensor
|
||||
# [num_reqs] CPU upper bound on seq_lens (see CommonAttentionMetadata).
|
||||
seq_lens_cpu_upper_bound: torch.Tensor
|
||||
# [num_reqs]
|
||||
dcp_local_seq_lens: torch.Tensor | None
|
||||
|
||||
@@ -121,6 +123,8 @@ class InputBatch:
|
||||
logits_indices = query_start_loc[1:] - 1
|
||||
cu_num_logits = torch.arange(num_reqs + 1, device=device, dtype=torch.int32)
|
||||
cu_num_logits_np = np.arange(num_reqs + 1, dtype=np.int32)
|
||||
# Dummy: seq_len == query_len (fresh-prefill shape).
|
||||
seq_lens_cpu_upper_bound = torch.from_numpy(num_scheduled_tokens.copy())
|
||||
return cls(
|
||||
req_ids=req_ids,
|
||||
num_reqs=num_reqs,
|
||||
@@ -136,6 +140,7 @@ class InputBatch:
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_np=query_start_loc_np,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=None,
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
|
||||
Reference in New Issue
Block a user