[Perf][Attention] Pin MLA chunked-context metadata tensors so H2D copies are truly non-blocking (#45074)

Signed-off-by: zixi-qi <zixi@inferact.ai>
This commit is contained in:
qizixi
2026-06-10 15:56:49 -07:00
committed by GitHub
parent 82d6b59f04
commit e2db0222e9
@@ -1664,7 +1664,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
.unsqueeze(1)
.expand(-1, num_prefills)
* max_context_chunk
)
).pin_memory()
chunk_ends = torch.min(
context_lens_cpu.unsqueeze(0), chunk_starts + max_context_chunk
)
@@ -1680,7 +1680,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
max_token_num_over_chunk = chunk_total_token.max().item()
token_to_seq_tensor_cpu = torch.zeros(
[num_chunks, max_token_num_over_chunk], dtype=torch.int32
[num_chunks, max_token_num_over_chunk],
dtype=torch.int32,
pin_memory=True,
)
range_idx = torch.arange(num_prefills, dtype=torch.int32)
for i in range(num_chunks):
@@ -1724,7 +1726,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
.unsqueeze(1)
.expand(-1, num_prefills)
* padded_local_max_context_chunk_across_ranks
)
).pin_memory()
local_chunk_ends = torch.min(
padded_local_context_lens_cpu.unsqueeze(0),
local_chunk_starts