[Core] Avoid seq_lens_cpu GPU->CPU sync (#40654)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -90,15 +90,23 @@ def create_cross_attention_backend(
|
||||
assert new_metadata.encoder_seq_lens_cpu is not None
|
||||
max_encoder_len = int(new_metadata.encoder_seq_lens_cpu.max())
|
||||
new_metadata.max_seq_len = max_encoder_len
|
||||
# Any computed tokens indicated decode step>1 (no chunked prefill)
|
||||
num_cache_decodes = (
|
||||
(common_attn_metadata.num_computed_tokens_cpu > 0).sum().item()
|
||||
# Any computed tokens indicates decode step>1 (no chunked prefill).
|
||||
# The upper bound is exact for this `> 0` test - prefill rows have
|
||||
# num_computed == 0 and decode rows have num_computed > 0.
|
||||
query_lens_cpu = (
|
||||
common_attn_metadata.query_start_loc_cpu[1:]
|
||||
- common_attn_metadata.query_start_loc_cpu[:-1]
|
||||
)
|
||||
assert common_attn_metadata.seq_lens_cpu_upper_bound is not None
|
||||
num_computed_tokens_cpu = (
|
||||
common_attn_metadata.seq_lens_cpu_upper_bound - query_lens_cpu
|
||||
)
|
||||
num_cache_decodes = (num_computed_tokens_cpu > 0).sum().item()
|
||||
if num_cache_decodes > 0:
|
||||
# CrossAttn KV cache has already been populated on first decoder step,
|
||||
# skip slot_mapping calculation for requests that do not need
|
||||
# reshape_and_cache.
|
||||
num_tokens = common_attn_metadata.num_computed_tokens_cpu.numpy()
|
||||
num_tokens = num_computed_tokens_cpu.numpy()
|
||||
new_metadata.encoder_seq_lens_cpu = np.where(
|
||||
num_tokens > 0, 0, new_metadata.encoder_seq_lens_cpu
|
||||
)
|
||||
|
||||
@@ -1822,13 +1822,18 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
|
||||
prefill_metadata = None
|
||||
if num_prefills > 0:
|
||||
num_computed_tokens_cpu = (
|
||||
common_attn_metadata.compute_num_computed_tokens().cpu()
|
||||
)
|
||||
|
||||
reqs_start = num_decodes # prefill_start
|
||||
|
||||
context_lens_cpu = num_computed_tokens_cpu[reqs_start:num_reqs]
|
||||
# Upper bound is exact for prefill rows (no D2H sync).
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
prefill_query_lens_cpu = (
|
||||
query_start_loc_cpu[reqs_start + 1 : num_reqs + 1]
|
||||
- query_start_loc_cpu[reqs_start:num_reqs]
|
||||
)
|
||||
context_lens_cpu = (
|
||||
seq_lens_cpu[reqs_start:num_reqs] - prefill_query_lens_cpu
|
||||
)
|
||||
max_context_len_cpu = context_lens_cpu.max().item()
|
||||
num_prefills_with_context_cpu = (context_lens_cpu > 0).sum().item()
|
||||
prefill_query_start_loc = (
|
||||
|
||||
Reference in New Issue
Block a user