[MRV2] Buffer util simplifications (#56888)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -1,10 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import (
|
||||
OMP_NUM_THREADS_SET_BY_VLLM,
|
||||
async_tensor_h2d,
|
||||
available_cpu_count,
|
||||
common_broadcastable_dtype,
|
||||
current_stream,
|
||||
@@ -192,3 +194,26 @@ def test_runtime_threads_override_vllm_set_omp_num_threads(
|
||||
torch.set_num_threads(3)
|
||||
set_torch_threads_for_runtime()
|
||||
assert torch.get_num_threads() == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"device", ["cpu", "cuda"] if torch.cuda.is_available() else ["cpu"]
|
||||
)
|
||||
def test_async_tensor_h2d_staging(device):
|
||||
"""Staging must preserve values while normalizing dtype and layout."""
|
||||
# Non-contiguous source with a dtype conversion: would previously sync.
|
||||
src = torch.arange(12, dtype=torch.int64).reshape(3, 4).T
|
||||
result = async_tensor_h2d(src, device=device, dtype=torch.int32)
|
||||
assert result.dtype == torch.int32
|
||||
assert result.is_contiguous()
|
||||
assert torch.equal(result.cpu(), src.to(torch.int32))
|
||||
|
||||
# numpy and list sources still work.
|
||||
assert torch.equal(
|
||||
async_tensor_h2d(np.arange(4, dtype=np.int64), device=device),
|
||||
torch.arange(4, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
async_tensor_h2d([1, 2, 3], device=device, dtype=torch.int32),
|
||||
torch.tensor([1, 2, 3], dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
@@ -996,7 +996,7 @@ def test_pcp_plan_deinterleave_restores_global_order(
|
||||
|
||||
monkeypatch.setattr("vllm.v1.attention.backends.mla.indexer.PIN_MEMORY", False)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.indexer.async_copy_to_gpu",
|
||||
"vllm.v1.attention.backends.mla.indexer.async_tensor_h2d",
|
||||
lambda x, device: x.to(device),
|
||||
)
|
||||
scheduled = np.array(req_lens, dtype=np.int64)
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers, set_dummy_c
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
|
||||
|
||||
def _copy_to_cpu(value, out=None, device=None):
|
||||
def _copy_to_cpu(value, device=None, dtype=None, out=None):
|
||||
tensor = torch.from_numpy(value) if isinstance(value, np.ndarray) else value
|
||||
if out is not None:
|
||||
return out.copy_(tensor)
|
||||
@@ -83,7 +83,7 @@ def test_replicated_decode_piecewise_graph_padding(monkeypatch):
|
||||
device=torch.device("cpu"),
|
||||
dcp_world_size=1,
|
||||
)
|
||||
monkeypatch.setattr(pcp_manager_module, "async_copy_to_gpu", _copy_to_cpu)
|
||||
monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu)
|
||||
|
||||
segments_by_rank, per_rank_num_tokens = manager._build_batch_layout(
|
||||
num_scheduled_tokens=np.ones(3, dtype=np.int32),
|
||||
@@ -158,7 +158,7 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch):
|
||||
device=torch.device("cpu"),
|
||||
dcp_world_size=1,
|
||||
)
|
||||
monkeypatch.setattr(pcp_manager_module, "async_copy_to_gpu", _copy_to_cpu)
|
||||
monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu)
|
||||
|
||||
with pytest.raises(ValueError, match="smaller than the largest rank-local batch"):
|
||||
manager._build_batch_layout(
|
||||
|
||||
Reference in New Issue
Block a user