[XPU] Fix UVA weight offloading (non-pinned-tensor views and static Triton launcher) (#51770)
Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
This commit is contained in:
@@ -326,6 +326,30 @@ class XPUPlatform(Platform):
|
|||||||
)
|
)
|
||||||
setattr(pass_config, flag, False)
|
setattr(pass_config, flag, False)
|
||||||
|
|
||||||
|
# UVA-offloaded weights are host USM allocations, which Inductor's
|
||||||
|
# static Triton launcher rejects ("Pointer argument doesn't reference
|
||||||
|
# XPU device memory"). Fall back to Triton's own launcher. Remove once
|
||||||
|
# the released torch contains pytorch/pytorch#188240, which relaxes
|
||||||
|
# that check to any memory type known by the driver.
|
||||||
|
offload_config = vllm_config.offload_config
|
||||||
|
uva_offloading = offload_config.offload_backend == "uva" or (
|
||||||
|
offload_config.offload_backend == "auto"
|
||||||
|
and offload_config.prefetch.offload_group_size == 0
|
||||||
|
and offload_config.uva.cpu_offload_gb > 0
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
uva_offloading
|
||||||
|
and not envs.VLLM_WEIGHT_OFFLOADING_DISABLE_UVA
|
||||||
|
and compilation_config.mode != CompilationMode.NONE
|
||||||
|
):
|
||||||
|
compilation_config.inductor_compile_config.setdefault(
|
||||||
|
"use_static_cuda_launcher", False
|
||||||
|
)
|
||||||
|
logger.info_once(
|
||||||
|
"Disabling Inductor's static Triton launcher because UVA "
|
||||||
|
"weight offloading is enabled."
|
||||||
|
)
|
||||||
|
|
||||||
# check and update parallel config
|
# check and update parallel config
|
||||||
parallel_config = vllm_config.parallel_config
|
parallel_config = vllm_config.parallel_config
|
||||||
# Only override worker_cls if it's still the default "auto"
|
# Only override worker_cls if it's still the default "auto"
|
||||||
|
|||||||
@@ -887,7 +887,15 @@ def get_accelerator_view_from_cpu_tensor(cpu_tensor: torch.Tensor) -> torch.Tens
|
|||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
if current_platform.is_xpu():
|
if current_platform.is_xpu():
|
||||||
assert cpu_tensor.is_pinned(), "CPU tensor must be pinned"
|
# Remove once the vllm-xpu-kernels fix for empty and non-pinned inputs
|
||||||
|
# (vllm-project/vllm-xpu-kernels#513) is in a released package.
|
||||||
|
if cpu_tensor.numel() == 0:
|
||||||
|
return torch.empty(cpu_tensor.shape, dtype=cpu_tensor.dtype, device="xpu")
|
||||||
|
if not cpu_tensor.is_pinned():
|
||||||
|
contiguous_cpu = cpu_tensor.contiguous()
|
||||||
|
pinned = torch.empty_like(contiguous_cpu, pin_memory=True)
|
||||||
|
pinned.copy_(contiguous_cpu)
|
||||||
|
cpu_tensor = pinned
|
||||||
return torch.ops._C.get_xpu_view_from_cpu_tensor(cpu_tensor)
|
return torch.ops._C.get_xpu_view_from_cpu_tensor(cpu_tensor)
|
||||||
elif current_platform.is_cuda_alike():
|
elif current_platform.is_cuda_alike():
|
||||||
return torch.ops._C.get_cuda_view_from_cpu_tensor(cpu_tensor)
|
return torch.ops._C.get_cuda_view_from_cpu_tensor(cpu_tensor)
|
||||||
|
|||||||
Reference in New Issue
Block a user