[Core] Avoid seq_lens_cpu GPU->CPU sync (#40654)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-04-24 00:35:55 +00:00
committed by GitHub
parent 62b1bbe470
commit fe85a92e86
19 changed files with 142 additions and 26 deletions
+2
View File
@@ -2155,6 +2155,7 @@ class GPUModelRunner(
:num_reqs_padded
]
seq_lens_cpu = self.optimistic_seq_lens_cpu[:num_reqs_padded]
seq_lens_cpu_upper_bound = seq_lens_cpu
# is_prefilling: True if request is still in prefill phase.
# Used by mamba backends to distinguish actual decodes from
@@ -2172,6 +2173,7 @@ class GPUModelRunner(
seq_lens=self.seq_lens[:num_reqs_padded],
_seq_lens_cpu=seq_lens_cpu,
_num_computed_tokens_cpu=num_computed_tokens_cpu,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
num_reqs=num_reqs_padded,
num_actual_tokens=num_tokens_padded,
max_query_len=max_query_len,