[Core] Avoid seq_lens_cpu GPU->CPU sync (#40654)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -177,7 +177,22 @@ def _make_metadata_with_slice(
|
||||
query_start_loc[1:] -= tokens_skipped
|
||||
query_start_loc_cpu[1:] -= tokens_skipped
|
||||
seq_lens = attn_metadata.seq_lens[request_slice]
|
||||
seq_lens_cpu = attn_metadata.seq_lens_cpu[request_slice]
|
||||
# Read raw fields to avoid triggering the deprecated D2H-syncing properties.
|
||||
seq_lens_cpu = (
|
||||
attn_metadata._seq_lens_cpu[request_slice]
|
||||
if attn_metadata._seq_lens_cpu is not None
|
||||
else None
|
||||
)
|
||||
seq_lens_cpu_upper_bound = (
|
||||
attn_metadata.seq_lens_cpu_upper_bound[request_slice]
|
||||
if attn_metadata.seq_lens_cpu_upper_bound is not None
|
||||
else None
|
||||
)
|
||||
num_computed_tokens_cpu = (
|
||||
attn_metadata._num_computed_tokens_cpu[request_slice]
|
||||
if attn_metadata._num_computed_tokens_cpu is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if splits_last_request:
|
||||
# NOTE: We use start_locs (the original query_start_loc_cpu) to calculate
|
||||
@@ -190,12 +205,16 @@ def _make_metadata_with_slice(
|
||||
# Make sure we don't modify the seq_lens tensors
|
||||
# (not cudagraph compatible)
|
||||
seq_lens = seq_lens.clone()
|
||||
seq_lens_cpu = seq_lens_cpu.clone()
|
||||
seq_lens[-1] -= tokens_skipped
|
||||
seq_lens_cpu[-1] -= tokens_skipped
|
||||
if seq_lens_cpu is not None:
|
||||
seq_lens_cpu = seq_lens_cpu.clone()
|
||||
seq_lens_cpu[-1] -= tokens_skipped
|
||||
if seq_lens_cpu_upper_bound is not None:
|
||||
seq_lens_cpu_upper_bound = seq_lens_cpu_upper_bound.clone()
|
||||
seq_lens_cpu_upper_bound[-1] -= tokens_skipped
|
||||
|
||||
max_seq_len = int(seq_lens_cpu.max())
|
||||
num_computed_tokens_cpu = attn_metadata.num_computed_tokens_cpu[request_slice]
|
||||
assert seq_lens_cpu_upper_bound is not None
|
||||
max_seq_len = int(seq_lens_cpu_upper_bound.max())
|
||||
|
||||
num_requests = request_slice.stop - request_slice.start
|
||||
num_actual_tokens = token_slice.stop - token_slice.start
|
||||
@@ -221,6 +240,7 @@ def _make_metadata_with_slice(
|
||||
max_seq_len=max_seq_len,
|
||||
block_table_tensor=block_table_tensor,
|
||||
slot_mapping=slot_mapping,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
_seq_lens_cpu=seq_lens_cpu,
|
||||
_num_computed_tokens_cpu=num_computed_tokens_cpu,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user