[Bugfix][ROCm] Use batch DMA for CPU KV cache loads (#49843)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
7f599d7854
commit
394beb633b
@@ -7,6 +7,7 @@ import uuid
|
|||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
|
from vllm import _custom_ops as ops
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.utils.math_utils import round_up
|
from vllm.utils.math_utils import round_up
|
||||||
from vllm.utils.torch_utils import set_random_seed
|
from vllm.utils.torch_utils import set_random_seed
|
||||||
@@ -16,6 +17,7 @@ from vllm.v1.kv_offload.base import (
|
|||||||
CanonicalKVCacheTensor,
|
CanonicalKVCacheTensor,
|
||||||
GPULoadStoreSpec,
|
GPULoadStoreSpec,
|
||||||
)
|
)
|
||||||
|
from vllm.v1.kv_offload.cpu import gpu_worker
|
||||||
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||||
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
||||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||||
@@ -32,6 +34,18 @@ NUM_MAPPINGS = [3]
|
|||||||
NUM_MAPPINGS_PER_GROUP = [2]
|
NUM_MAPPINGS_PER_GROUP = [2]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific test")
|
||||||
|
def test_rocm_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(gpu_worker, "HAS_TRITON", True)
|
||||||
|
monkeypatch.setattr(gpu_worker.current_platform, "is_xpu", lambda: False)
|
||||||
|
monkeypatch.setattr(gpu_worker.current_platform, "is_rocm", lambda: True)
|
||||||
|
|
||||||
|
refs = [[CanonicalKVCacheRef(tensor_idx=0, page_size_bytes=512)]]
|
||||||
|
assert gpu_worker._select_swap_blocks_fn(refs, gpu_to_cpu=False) is (
|
||||||
|
ops.swap_blocks_batch
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
|
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
|
||||||
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
|
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
|
||||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||||
|
|||||||
@@ -41,10 +41,10 @@ def _select_swap_blocks_fn(
|
|||||||
if gpu_to_cpu:
|
if gpu_to_cpu:
|
||||||
return ops.swap_blocks_batch
|
return ops.swap_blocks_batch
|
||||||
# Fall back to the C++ DMA path on platforms where Triton isn't usable
|
# Fall back to the C++ DMA path on platforms where Triton isn't usable
|
||||||
# (e.g. ROCm builds without Triton) or where GPU kernels cannot directly
|
# (e.g. ROCm host mappings) or where GPU kernels cannot directly
|
||||||
# dereference CPU pointers (XPU lacks CUDA's unified virtual address space,
|
# dereference CPU pointers (XPU lacks CUDA's unified virtual address space,
|
||||||
# so the Triton kernel's tl.load(cpu_ptr) is invalid on XPU).
|
# so the Triton kernel's tl.load(cpu_ptr) is invalid on XPU).
|
||||||
if not HAS_TRITON or current_platform.is_xpu():
|
if not HAS_TRITON or current_platform.is_xpu() or current_platform.is_rocm():
|
||||||
return ops.swap_blocks_batch
|
return ops.swap_blocks_batch
|
||||||
page_sizes = [r.page_size_bytes for g in kv_cache_groups_data_refs for r in g]
|
page_sizes = [r.page_size_bytes for g in kv_cache_groups_data_refs for r in g]
|
||||||
# Triton wins only on small, 8-byte-aligned payloads.
|
# Triton wins only on small, 8-byte-aligned payloads.
|
||||||
|
|||||||
Reference in New Issue
Block a user