[Perf] Ensure async h2d copies are pinned in more places (#55202)
This commit is contained in:
@@ -265,6 +265,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.math_utils import cdiv, round_down, round_up
|
||||
from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
@@ -2510,8 +2511,13 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
query_start_loc_cpu[reqs_start + 1 : num_reqs + 1]
|
||||
- query_start_loc_cpu[reqs_start:num_reqs]
|
||||
)
|
||||
context_lens_cpu = (
|
||||
seq_lens_cpu[reqs_start:num_reqs] - prefill_query_lens_cpu
|
||||
context_lens_cpu = torch.empty(
|
||||
num_prefills, dtype=seq_lens_cpu.dtype, pin_memory=PIN_MEMORY
|
||||
)
|
||||
torch.subtract(
|
||||
seq_lens_cpu[reqs_start:num_reqs],
|
||||
prefill_query_lens_cpu,
|
||||
out=context_lens_cpu,
|
||||
)
|
||||
prefill_query_start_loc = (
|
||||
query_start_loc[reqs_start:] - query_start_loc[reqs_start]
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.worker.ubatching import (
|
||||
dbo_enabled,
|
||||
dbo_maybe_run_recv_hook,
|
||||
@@ -109,7 +110,10 @@ class ExpertTokensMetadata:
|
||||
expert_num_tokens_list: list[int], device: str
|
||||
) -> "ExpertTokensMetadata":
|
||||
expert_num_tokens_cpu = torch.tensor(
|
||||
expert_num_tokens_list, device="cpu", dtype=torch.int32
|
||||
expert_num_tokens_list,
|
||||
device="cpu",
|
||||
dtype=torch.int32,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
return ExpertTokensMetadata(
|
||||
expert_num_tokens=expert_num_tokens_cpu.to(device, non_blocking=True),
|
||||
|
||||
Reference in New Issue
Block a user