[Core] Avoid seq_lens_cpu GPU->CPU sync (#40654)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -173,6 +173,12 @@ class DefaultModelState(ModelState):
|
||||
num_tokens = input_batch.num_tokens
|
||||
query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np)
|
||||
max_query_len = input_batch.num_scheduled_tokens.max().item()
|
||||
seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound
|
||||
if for_capture:
|
||||
# Capture with worst-case max_seq_len so the graph is valid at any replay.
|
||||
max_seq_len = self.max_model_len
|
||||
else:
|
||||
max_seq_len = int(seq_lens_cpu_upper_bound[:num_reqs].max().item())
|
||||
attn_metadata = build_attn_metadata(
|
||||
attn_groups=attn_groups,
|
||||
num_reqs=num_reqs,
|
||||
@@ -181,10 +187,11 @@ class DefaultModelState(ModelState):
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=max_query_len,
|
||||
seq_lens=input_batch.seq_lens,
|
||||
max_seq_len=self.max_model_len,
|
||||
max_seq_len=max_seq_len,
|
||||
block_tables=block_tables,
|
||||
slot_mappings=slot_mappings,
|
||||
kv_cache_config=kv_cache_config,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=input_batch.dcp_local_seq_lens,
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
Reference in New Issue
Block a user