[Core] Avoid seq_lens_cpu GPU->CPU sync (#40654)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-04-24 00:35:55 +00:00
committed by GitHub
parent 62b1bbe470
commit fe85a92e86
19 changed files with 142 additions and 26 deletions
+7
View File
@@ -151,6 +151,12 @@ class DFlashProposer(SpecDecodeBaseProposer):
if has_num_rejected:
effective_seq_lens = effective_seq_lens - num_rejected_tokens_gpu
# Skip num_rejected_tokens (GPU-only); overestimating is fine here.
new_seq_lens_cpu_upper_bound = (
cad.seq_lens_cpu_upper_bound + num_query_per_req
if cad.seq_lens_cpu_upper_bound is not None
else None
)
new_cad = CommonAttentionMetadata(
query_start_loc=new_query_start_loc,
seq_lens=effective_seq_lens + num_query_per_req,
@@ -160,6 +166,7 @@ class DFlashProposer(SpecDecodeBaseProposer):
),
_seq_lens_cpu=None,
_num_computed_tokens_cpu=None,
seq_lens_cpu_upper_bound=new_seq_lens_cpu_upper_bound,
num_reqs=cad.num_reqs,
num_actual_tokens=num_query_total,
max_query_len=num_query_per_req,