[Core] Avoid seq_lens_cpu GPU->CPU sync (#40654)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-04-24 00:35:55 +00:00
committed by GitHub
parent 62b1bbe470
commit fe85a92e86
19 changed files with 142 additions and 26 deletions
+25 -5
View File
@@ -177,7 +177,22 @@ def _make_metadata_with_slice(
query_start_loc[1:] -= tokens_skipped
query_start_loc_cpu[1:] -= tokens_skipped
seq_lens = attn_metadata.seq_lens[request_slice]
seq_lens_cpu = attn_metadata.seq_lens_cpu[request_slice]
# Read raw fields to avoid triggering the deprecated D2H-syncing properties.
seq_lens_cpu = (
attn_metadata._seq_lens_cpu[request_slice]
if attn_metadata._seq_lens_cpu is not None
else None
)
seq_lens_cpu_upper_bound = (
attn_metadata.seq_lens_cpu_upper_bound[request_slice]
if attn_metadata.seq_lens_cpu_upper_bound is not None
else None
)
num_computed_tokens_cpu = (
attn_metadata._num_computed_tokens_cpu[request_slice]
if attn_metadata._num_computed_tokens_cpu is not None
else None
)
if splits_last_request:
# NOTE: We use start_locs (the original query_start_loc_cpu) to calculate
@@ -190,12 +205,16 @@ def _make_metadata_with_slice(
# Make sure we don't modify the seq_lens tensors
# (not cudagraph compatible)
seq_lens = seq_lens.clone()
seq_lens_cpu = seq_lens_cpu.clone()
seq_lens[-1] -= tokens_skipped
seq_lens_cpu[-1] -= tokens_skipped
if seq_lens_cpu is not None:
seq_lens_cpu = seq_lens_cpu.clone()
seq_lens_cpu[-1] -= tokens_skipped
if seq_lens_cpu_upper_bound is not None:
seq_lens_cpu_upper_bound = seq_lens_cpu_upper_bound.clone()
seq_lens_cpu_upper_bound[-1] -= tokens_skipped
max_seq_len = int(seq_lens_cpu.max())
num_computed_tokens_cpu = attn_metadata.num_computed_tokens_cpu[request_slice]
assert seq_lens_cpu_upper_bound is not None
max_seq_len = int(seq_lens_cpu_upper_bound.max())
num_requests = request_slice.stop - request_slice.start
num_actual_tokens = token_slice.stop - token_slice.start
@@ -221,6 +240,7 @@ def _make_metadata_with_slice(
max_seq_len=max_seq_len,
block_table_tensor=block_table_tensor,
slot_mapping=slot_mapping,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
_seq_lens_cpu=seq_lens_cpu,
_num_computed_tokens_cpu=num_computed_tokens_cpu,
)