[Core] Avoid seq_lens_cpu GPU->CPU sync (#40654)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -227,12 +227,15 @@ def build_attn_metadata(
|
||||
block_tables: Sequence[torch.Tensor],
|
||||
slot_mappings: torch.Tensor,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
seq_lens_cpu_upper_bound: torch.Tensor | None = None,
|
||||
dcp_local_seq_lens: torch.Tensor | None = None,
|
||||
encoder_seq_lens: dict[int, tuple[torch.Tensor, np.ndarray]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
seq_lens = seq_lens[:num_reqs]
|
||||
if dcp_local_seq_lens is not None:
|
||||
dcp_local_seq_lens = dcp_local_seq_lens[:num_reqs]
|
||||
if seq_lens_cpu_upper_bound is not None:
|
||||
seq_lens_cpu_upper_bound = seq_lens_cpu_upper_bound[:num_reqs]
|
||||
|
||||
attn_metadata: dict[str, Any] = {}
|
||||
num_kv_cache_groups = len(kv_cache_config.kv_cache_groups)
|
||||
@@ -244,6 +247,7 @@ def build_attn_metadata(
|
||||
query_start_loc=query_start_loc_gpu,
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
max_seq_len=max_seq_len,
|
||||
num_reqs=num_reqs,
|
||||
num_actual_tokens=num_tokens,
|
||||
|
||||
Reference in New Issue
Block a user