[Core] Ensure memory is pinned prior to async h2d copy (#45424)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -46,8 +46,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput
|
||||
@@ -498,7 +497,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
"""Build KV-block zeroing metadata; invoked from gpu_worker."""
|
||||
self.kv_block_zeroer = KVBlockZeroer(
|
||||
self.device,
|
||||
is_pin_memory_available(),
|
||||
pin_memory=PIN_MEMORY,
|
||||
attn_groups_iter=(g for groups in self.attn_groups for g in groups),
|
||||
kernel_block_sizes=self.kernel_block_sizes,
|
||||
cache_dtype=self.cache_config.cache_dtype,
|
||||
|
||||
Reference in New Issue
Block a user