[Perf] Ensure async h2d copies are pinned in more places (#55202)

This commit is contained in:
Nick Hill
2026-09-04 17:17:01 -07:00
committed by GitHub
parent 874df9373d
commit 8277c42e4c
21 changed files with 206 additions and 73 deletions
+6 -2
View File
@@ -70,6 +70,7 @@ from vllm.model_executor.models.glm4_1v import (
Glm4vVisionTransformer,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.utils.torch_utils import PIN_MEMORY
from .utils import (
maybe_prefix,
@@ -344,8 +345,11 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
cu_seqlens = torch.repeat_interleave(
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
).cumsum(dim=0, dtype=torch.int32)
cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)
# cat straight into a pinned buffer so the H2D copy stays non-blocking.
cu_seqlens = torch.cat(
[cu_seqlens.new_zeros(1), cu_seqlens],
out=cu_seqlens.new_empty(cu_seqlens.numel() + 1, pin_memory=PIN_MEMORY),
).to(self.device, non_blocking=True)
# pre-compute max_seqlen for attn mask to reduce cuMemcpy operations
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)