[Perf] Ensure async h2d copies are pinned in more places (#55202)

This commit is contained in:
Nick Hill
2026-09-04 17:17:01 -07:00
committed by GitHub
parent 874df9373d
commit 8277c42e4c
21 changed files with 206 additions and 73 deletions
@@ -265,6 +265,7 @@ from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.math_utils import cdiv, round_down, round_up
from vllm.utils.torch_utils import (
PIN_MEMORY,
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
@@ -2510,8 +2511,13 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
query_start_loc_cpu[reqs_start + 1 : num_reqs + 1]
- query_start_loc_cpu[reqs_start:num_reqs]
)
context_lens_cpu = (
seq_lens_cpu[reqs_start:num_reqs] - prefill_query_lens_cpu
context_lens_cpu = torch.empty(
num_prefills, dtype=seq_lens_cpu.dtype, pin_memory=PIN_MEMORY
)
torch.subtract(
seq_lens_cpu[reqs_start:num_reqs],
prefill_query_lens_cpu,
out=context_lens_cpu,
)
prefill_query_start_loc = (
query_start_loc[reqs_start:] - query_start_loc[reqs_start]
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.worker.ubatching import (
dbo_enabled,
dbo_maybe_run_recv_hook,
@@ -109,7 +110,10 @@ class ExpertTokensMetadata:
expert_num_tokens_list: list[int], device: str
) -> "ExpertTokensMetadata":
expert_num_tokens_cpu = torch.tensor(
expert_num_tokens_list, device="cpu", dtype=torch.int32
expert_num_tokens_list,
device="cpu",
dtype=torch.int32,
pin_memory=PIN_MEMORY,
)
return ExpertTokensMetadata(
expert_num_tokens=expert_num_tokens_cpu.to(device, non_blocking=True),