[Core][Kernel] Share persistent workspaces for Marlin and Humming (#57421)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: ErinYin <liang.yin@daocloud.io>
This commit is contained in:
@@ -1384,24 +1384,33 @@ def test_humming_global_valid_shape_m(
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_humming_permute_scratch_is_keyed_by_runtime_topk(
|
||||
def test_humming_permute_scratch_is_shared_by_config(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
|
||||
import vllm.model_executor.layers.fused_moe.moe_permute_unpermute as permute
|
||||
import vllm.v1.worker.workspace as workspace
|
||||
|
||||
manager = workspace.WorkspaceManager(torch.device("cpu"))
|
||||
monkeypatch.setattr(workspace, "_manager", manager)
|
||||
|
||||
scratch_topk6 = Mock()
|
||||
scratch_topk1 = Mock()
|
||||
scratch_type = Mock(side_effect=[scratch_topk6, scratch_topk1])
|
||||
monkeypatch.setattr(humming, "moe_permute_unpermute_supported", lambda: True)
|
||||
monkeypatch.setattr(humming, "MoEPermuteScratch", scratch_type)
|
||||
monkeypatch.setattr(permute, "MoEPermuteScratch", scratch_type)
|
||||
moe_config = make_dummy_moe_config(max_num_tokens=512, experts_per_token=6)
|
||||
moe_config.moe_parallel_config.dp_size = 2
|
||||
experts = SimpleNamespace(_permute_scratch={}, moe_config=moe_config)
|
||||
experts = SimpleNamespace(moe_config=moe_config)
|
||||
other_layer = SimpleNamespace(moe_config=moe_config)
|
||||
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
|
||||
assert (
|
||||
humming.HummingExpertsBase._get_permute_scratch(other_layer, 6) is scratch_topk6
|
||||
)
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 1) is scratch_topk1
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
|
||||
|
||||
@@ -1412,6 +1421,11 @@ def test_humming_permute_scratch_is_keyed_by_runtime_topk(
|
||||
assert second_call.kwargs["max_num_tokens"] == 6144
|
||||
assert second_call.kwargs["topk"] == 1
|
||||
|
||||
manager.lock()
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
|
||||
with pytest.raises(AssertionError, match="was not allocated during warmup"):
|
||||
humming.HummingExpertsBase._get_permute_scratch(experts, 2)
|
||||
|
||||
|
||||
def test_humming_delegates_to_instance_activation():
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -15,12 +15,14 @@ from vllm.model_executor.layers.fused_moe.expert_map_manager import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
|
||||
MoEPermuteScratch,
|
||||
get_moe_permute_scratch,
|
||||
moe_permute,
|
||||
moe_permute_unpermute_supported,
|
||||
moe_unpermute,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
NUM_EXPERTS = [16, 64, 256]
|
||||
TOP_KS = [2, 6, 8]
|
||||
@@ -120,6 +122,7 @@ def torch_unpermute(
|
||||
@pytest.mark.parametrize("topk", TOP_KS)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("ep_size", EP_SIZE)
|
||||
@pytest.mark.parametrize("use_scratch", [False, True])
|
||||
def test_moe_permute_unpermute(
|
||||
n_token: int,
|
||||
n_hidden: int,
|
||||
@@ -127,6 +130,8 @@ def test_moe_permute_unpermute(
|
||||
n_expert: int,
|
||||
ep_size: int,
|
||||
dtype: torch.dtype,
|
||||
use_scratch: bool,
|
||||
workspace_init,
|
||||
):
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
@@ -160,6 +165,18 @@ def test_moe_permute_unpermute(
|
||||
expert_map=expert_map,
|
||||
)
|
||||
|
||||
scratch = None
|
||||
if use_scratch:
|
||||
scratch = get_moe_permute_scratch(
|
||||
max_num_tokens=n_token,
|
||||
topk=topk,
|
||||
num_experts=n_expert,
|
||||
num_local_experts=n_local_expert,
|
||||
device=hidden_states.device,
|
||||
hidden_size=n_hidden,
|
||||
hidden_dtype=dtype,
|
||||
)
|
||||
|
||||
(
|
||||
permuted_hidden_states,
|
||||
_,
|
||||
@@ -173,6 +190,7 @@ def test_moe_permute_unpermute(
|
||||
n_expert=n_expert,
|
||||
n_local_expert=n_local_expert,
|
||||
expert_map=expert_map,
|
||||
scratch=scratch,
|
||||
)
|
||||
|
||||
# check expert_first_token_offset
|
||||
@@ -212,20 +230,22 @@ def test_moe_permute_unpermute(
|
||||
torch.testing.assert_close(result4, gold4, atol=2e-2, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
def test_moe_permute_reuses_scratch_buffers(dtype: torch.dtype):
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("n_token", [1, 33, 128])
|
||||
@pytest.mark.parametrize("topk", [1, 6, 8])
|
||||
def test_moe_permute_reuses_scratch_buffers(
|
||||
dtype: torch.dtype, n_token: int, topk: int, workspace_init
|
||||
):
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
n_token = 64
|
||||
n_hidden = 2048
|
||||
n_expert = 16
|
||||
topk = 4
|
||||
hidden_states = torch.randn((n_token, n_hidden), device="cuda").to(dtype)
|
||||
gating_output = torch.randn((n_token, n_expert), device="cuda").to(dtype)
|
||||
_, topk_ids, _ = fused_topk(hidden_states, gating_output, topk, False)
|
||||
|
||||
scratch = MoEPermuteScratch(
|
||||
scratch_config = dict(
|
||||
max_num_tokens=n_token,
|
||||
topk=topk,
|
||||
num_experts=n_expert,
|
||||
@@ -234,6 +254,7 @@ def test_moe_permute_reuses_scratch_buffers(dtype: torch.dtype):
|
||||
hidden_size=n_hidden,
|
||||
hidden_dtype=hidden_states.dtype,
|
||||
)
|
||||
scratch = get_moe_permute_scratch(**scratch_config)
|
||||
|
||||
first = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
@@ -242,13 +263,26 @@ def test_moe_permute_reuses_scratch_buffers(dtype: torch.dtype):
|
||||
n_expert=n_expert,
|
||||
scratch=scratch,
|
||||
)
|
||||
second = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=None,
|
||||
topk_ids=topk_ids,
|
||||
n_expert=n_expert,
|
||||
scratch=scratch,
|
||||
)
|
||||
current_workspace_manager().lock()
|
||||
assert get_moe_permute_scratch(**scratch_config) is scratch
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
second = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=None,
|
||||
topk_ids=topk_ids,
|
||||
n_expert=n_expert,
|
||||
scratch=get_moe_permute_scratch(**scratch_config),
|
||||
)
|
||||
|
||||
for _ in range(2):
|
||||
hidden_states.add_(1)
|
||||
topk_ids.copy_(topk_ids.roll(1, 0))
|
||||
graph.replay()
|
||||
expected = moe_permute(hidden_states, None, topk_ids, n_expert)
|
||||
for actual, reference in zip(second, expected):
|
||||
if actual is not None:
|
||||
torch.testing.assert_close(actual, reference)
|
||||
|
||||
(
|
||||
permuted_hidden_states_1,
|
||||
@@ -288,6 +322,123 @@ def test_moe_permute_reuses_scratch_buffers(dtype: torch.dtype):
|
||||
)
|
||||
|
||||
|
||||
def test_moe_permute_scratch_reused_across_graph_sizes(workspace_init) -> None:
|
||||
"""Switching graph sizes must not expose metadata left by a larger batch."""
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
scratch = get_moe_permute_scratch(
|
||||
max_num_tokens=128,
|
||||
topk=6,
|
||||
num_experts=16,
|
||||
num_local_experts=16,
|
||||
device=device,
|
||||
hidden_size=2048,
|
||||
hidden_dtype=torch.bfloat16,
|
||||
)
|
||||
current_workspace_manager().lock()
|
||||
runs = []
|
||||
for n_token in (128, 1, 33):
|
||||
hidden = torch.randn((n_token, 2048), dtype=torch.bfloat16, device=device)
|
||||
topk_ids = torch.randint(16, (n_token, 6), device=device)
|
||||
moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
|
||||
runs.append((graph, hidden, topk_ids, output))
|
||||
|
||||
for graph, hidden, topk_ids, output in runs + runs[::-1]:
|
||||
hidden.add_(1)
|
||||
topk_ids.copy_((topk_ids + 1) % 16)
|
||||
graph.replay()
|
||||
expected = moe_permute(hidden, None, topk_ids, 16)
|
||||
for actual, reference in zip(output, expected):
|
||||
if actual is not None:
|
||||
torch.testing.assert_close(actual, reference)
|
||||
|
||||
|
||||
def test_moe_permute_scratch_isolated_across_execution_slots(monkeypatch) -> None:
|
||||
"""Concurrent graph replays in different ubatches/lanes cannot share scratch."""
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
from vllm.v1.worker import workspace
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
manager = workspace.WorkspaceManager(device, num_ubatches=2, num_lanes=2)
|
||||
monkeypatch.setattr(workspace, "_manager", manager)
|
||||
ubatch = 0
|
||||
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: ubatch)
|
||||
scratch_config = dict(
|
||||
max_num_tokens=64,
|
||||
topk=6,
|
||||
num_experts=16,
|
||||
num_local_experts=16,
|
||||
device=device,
|
||||
hidden_size=2048,
|
||||
hidden_dtype=torch.bfloat16,
|
||||
)
|
||||
runs = []
|
||||
for i, n_token in enumerate((1, 7, 33, 64)):
|
||||
ubatch = i // 2
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with workspace.use_workspace_lane(i % 2), torch.cuda.stream(stream):
|
||||
hidden = torch.randn((n_token, 2048), dtype=torch.bfloat16, device=device)
|
||||
topk_ids = torch.randint(16, (n_token, 6), device=device)
|
||||
scratch = get_moe_permute_scratch(**scratch_config)
|
||||
moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
output = moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
|
||||
runs.append((stream, graph, hidden, topk_ids, output, scratch))
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
|
||||
assert len({run[-1].permuted_hidden_states.data_ptr() for run in runs}) == 4
|
||||
manager.lock()
|
||||
for _ in range(3):
|
||||
for i, (stream, graph, hidden, topk_ids, _, scratch) in enumerate(runs):
|
||||
ubatch = i // 2
|
||||
with workspace.use_workspace_lane(i % 2), torch.cuda.stream(stream):
|
||||
assert get_moe_permute_scratch(**scratch_config) is scratch
|
||||
hidden.add_(1)
|
||||
topk_ids.copy_((topk_ids + 1) % 16)
|
||||
graph.replay()
|
||||
for stream, _, hidden, topk_ids, output, _ in runs:
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
expected = moe_permute(hidden, None, topk_ids, 16)
|
||||
for actual, reference in zip(output, expected):
|
||||
if actual is not None:
|
||||
torch.testing.assert_close(actual, reference)
|
||||
|
||||
|
||||
def test_moe_permute_scratch_without_manager(monkeypatch) -> None:
|
||||
"""Standalone calls get independent scratch with initialized row indices."""
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
from vllm.v1.worker import workspace
|
||||
|
||||
monkeypatch.setattr(workspace, "_manager", None)
|
||||
config = dict(
|
||||
max_num_tokens=4,
|
||||
topk=2,
|
||||
num_experts=4,
|
||||
num_local_experts=4,
|
||||
device=torch.device("cuda"),
|
||||
)
|
||||
first = get_moe_permute_scratch(**config)
|
||||
second = get_moe_permute_scratch(**config)
|
||||
|
||||
assert (
|
||||
first.token_expert_indices.data_ptr() != second.token_expert_indices.data_ptr()
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
first.token_expert_indices, torch.arange(8, dtype=torch.int32, device="cuda")
|
||||
)
|
||||
|
||||
|
||||
def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
per_token_quant_int8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
get_marlin_workspace,
|
||||
marlin_make_empty,
|
||||
marlin_make_workspace_new,
|
||||
marlin_permute_bias,
|
||||
@@ -43,6 +44,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
@@ -477,6 +479,62 @@ def test_marlin_gemm(
|
||||
assert max_diff < 0.04
|
||||
|
||||
|
||||
def test_marlin_persistent_locks_cuda_graph_streams(workspace_init):
|
||||
"""Compiled GEMMs select stream-local locks at runtime and survive replay."""
|
||||
size_m, size_k, size_n = 32, 1024, 2048
|
||||
inputs = [rand_data((size_m, size_k)) for _ in range(2)]
|
||||
weight = rand_data((size_k, size_n))
|
||||
quant_type = scalar_types.uint4b8
|
||||
w_ref, qweight, scales = marlin_quantize(weight, quant_type, 128)
|
||||
zeros = marlin_make_empty(scales.device)
|
||||
streams = [torch.cuda.Stream() for _ in inputs]
|
||||
graphs = [torch.cuda.CUDAGraph() for _ in inputs]
|
||||
locks = []
|
||||
outputs = []
|
||||
|
||||
@torch.compile(backend="eager", fullgraph=True)
|
||||
def run(x):
|
||||
return torch.ops.vllm.marlin_gemm(
|
||||
x,
|
||||
qweight,
|
||||
None,
|
||||
scales,
|
||||
None,
|
||||
None,
|
||||
zeros,
|
||||
None,
|
||||
quant_type.id,
|
||||
size_m,
|
||||
size_n,
|
||||
size_k,
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
)
|
||||
|
||||
for stream, graph, x in zip(streams, graphs, inputs):
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
locks.append(get_marlin_workspace(x.device))
|
||||
run(x)
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
outputs.append(run(x))
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
|
||||
assert locks[0].data_ptr() != locks[1].data_ptr()
|
||||
current_workspace_manager().lock()
|
||||
for _ in range(3):
|
||||
for stream, graph, x, lock in zip(streams, graphs, inputs, locks):
|
||||
with torch.cuda.stream(stream):
|
||||
assert get_marlin_workspace(x.device) is lock
|
||||
x.add_(0.1)
|
||||
graph.replay()
|
||||
for stream in streams:
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
for x, output, lock in zip(inputs, outputs, locks):
|
||||
assert compute_max_diff(output, x @ w_ref) < 0.04
|
||||
assert torch.count_nonzero(lock) == 0
|
||||
|
||||
|
||||
def test_marlin_gemm_subset_input():
|
||||
quant_type = scalar_types.uint4b8
|
||||
group_size = 128
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
GPTQ_MARLIN_TILE,
|
||||
apply_gptq_marlin_linear,
|
||||
get_marlin_workspace,
|
||||
marlin_make_empty,
|
||||
marlin_make_workspace_new,
|
||||
marlin_moe_padded_intermediate,
|
||||
@@ -225,7 +226,7 @@ def test_fp8_marlin_padded_round_trip(shape, use_bias):
|
||||
input=x,
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale,
|
||||
workspace=layer.workspace,
|
||||
workspace=get_marlin_workspace(layer.weight.device),
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
bias=layer.bias if use_bias else None,
|
||||
@@ -288,7 +289,7 @@ def test_nvfp4_marlin_padded_round_trip(shape):
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale,
|
||||
weight_global_scale=layer.weight_global_scale,
|
||||
workspace=layer.workspace,
|
||||
workspace=get_marlin_workspace(layer.weight.device),
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
)
|
||||
@@ -392,7 +393,7 @@ def test_fp8_block_marlin_padded_round_trip(shape):
|
||||
input=x,
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale_inv,
|
||||
workspace=layer.workspace,
|
||||
workspace=get_marlin_workspace(layer.weight.device),
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
bias=None,
|
||||
@@ -440,7 +441,7 @@ def test_mxfp8_marlin_padded_round_trip(shape):
|
||||
input=x,
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale,
|
||||
workspace=layer.workspace,
|
||||
workspace=get_marlin_workspace(layer.weight.device),
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
)
|
||||
@@ -756,7 +757,7 @@ def test_fp8_marlin_moe_padded_round_trip(shape, quant):
|
||||
topk_ids,
|
||||
quant_type_id=scalar_types.float8_e4m3fn.id,
|
||||
global_num_experts=e,
|
||||
workspace=layer.workspace,
|
||||
workspace=get_marlin_workspace(device),
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
ref = torch_experts(
|
||||
@@ -833,7 +834,7 @@ def test_mxfp8_marlin_moe_padded_round_trip(shape):
|
||||
topk_ids,
|
||||
quant_type_id=scalar_types.float8_e4m3fn.id,
|
||||
global_num_experts=e,
|
||||
workspace=layer.workspace,
|
||||
workspace=get_marlin_workspace(device),
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
ref = torch_experts(
|
||||
|
||||
@@ -330,8 +330,8 @@ def _load_marlin_checkpoint_format_weights(layer):
|
||||
)
|
||||
|
||||
|
||||
def test_marlin_post_load_preserves_runtime_tensor_addresses(monkeypatch, dist_init):
|
||||
"""Marlin must reuse the workspace storage captured by CUDA graphs."""
|
||||
def test_marlin_post_load_does_not_own_workspace(monkeypatch, dist_init):
|
||||
"""Weight reload must not create layer-owned Marlin lock storage."""
|
||||
_stub_marlin_ops(monkeypatch)
|
||||
kernel = _make_marlin_kernel()
|
||||
|
||||
@@ -339,19 +339,17 @@ def test_marlin_post_load_preserves_runtime_tensor_addresses(monkeypatch, dist_i
|
||||
_load_marlin_checkpoint_format_weights(layer)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
workspace_ptr = kernel.workspace.data_ptr()
|
||||
assert not hasattr(kernel, "workspace")
|
||||
|
||||
_load_marlin_checkpoint_format_weights(layer)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
assert kernel.workspace.data_ptr() == workspace_ptr
|
||||
assert torch.all(kernel.workspace == 0)
|
||||
assert not hasattr(kernel, "workspace")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("variant", ["fp8", "mxfp8", "nvfp4"])
|
||||
def test_marlin_prepare_layer_preserves_workspace_address(monkeypatch, variant):
|
||||
"""The Marlin fallback prepare_* functions rerun on weight reload and must
|
||||
reuse the workspace storage whose address captured CUDA graphs hold."""
|
||||
def test_marlin_prepare_layer_does_not_own_workspace(monkeypatch, variant):
|
||||
"""Weight preparation must not attach runtime workspace to model layers."""
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils import (
|
||||
marlin_utils,
|
||||
@@ -417,34 +415,57 @@ def test_marlin_prepare_layer_preserves_workspace_address(monkeypatch, variant):
|
||||
|
||||
load_checkpoint_format_weights()
|
||||
prepare(layer)
|
||||
workspace_ptr = layer.workspace.data_ptr()
|
||||
assert not hasattr(layer, "workspace")
|
||||
|
||||
# Reload: fresh checkpoint-format tensors, prepare runs again
|
||||
load_checkpoint_format_weights()
|
||||
prepare(layer)
|
||||
|
||||
assert layer.workspace.data_ptr() == workspace_ptr
|
||||
assert torch.all(layer.workspace == 0)
|
||||
assert not hasattr(layer, "workspace")
|
||||
|
||||
|
||||
def test_marlin_make_workspace_new_rejects_incompatible_existing(monkeypatch):
|
||||
"""An incompatible existing workspace means the address captured by CUDA
|
||||
graphs is already unusable; allocating a replacement would hide that."""
|
||||
def test_marlin_workspace_uses_persistent_workspace_manager(monkeypatch):
|
||||
"""Calls reuse initialized locks; independent streams get separate storage."""
|
||||
from vllm.model_executor.layers.quantization.utils import marlin_utils
|
||||
from vllm.utils import torch_utils
|
||||
from vllm.v1.worker import workspace as workspace_module
|
||||
|
||||
monkeypatch.setattr(marlin_utils, "num_compute_units", lambda _: 4)
|
||||
device = torch.device("cpu")
|
||||
manager = workspace_module.WorkspaceManager(device)
|
||||
monkeypatch.setattr(workspace_module, "_manager", manager)
|
||||
stream = "main"
|
||||
monkeypatch.setattr(torch_utils, "current_stream", lambda: stream)
|
||||
|
||||
workspace = marlin_utils.marlin_make_workspace_new(device)
|
||||
reused = marlin_utils.marlin_make_workspace_new(device, existing=workspace)
|
||||
assert reused is workspace
|
||||
workspace = marlin_utils.get_marlin_workspace(device)
|
||||
assert workspace.shape == (4 * marlin_utils.MARLIN_MAX_BLOCKS_PER_SM,)
|
||||
assert workspace.dtype == torch.int32
|
||||
assert torch.count_nonzero(workspace) == 0
|
||||
|
||||
with pytest.raises(ValueError, match="incompatible"):
|
||||
marlin_utils.marlin_make_workspace_new(device, 4, existing=workspace)
|
||||
with pytest.raises(ValueError, match="incompatible"):
|
||||
marlin_utils.marlin_make_workspace_new(
|
||||
device, existing=workspace.to(torch.int64)
|
||||
)
|
||||
workspace.fill_(1)
|
||||
assert marlin_utils.get_marlin_workspace(device) is workspace
|
||||
assert torch.all(workspace == 1)
|
||||
|
||||
stream = "aux"
|
||||
aux_workspace = marlin_utils.get_marlin_workspace(device)
|
||||
assert aux_workspace.data_ptr() != workspace.data_ptr()
|
||||
assert torch.count_nonzero(aux_workspace) == 0
|
||||
|
||||
manager.lock()
|
||||
assert marlin_utils.get_marlin_workspace(device) is aux_workspace
|
||||
|
||||
|
||||
def test_marlin_workspace_without_manager(monkeypatch):
|
||||
from vllm.model_executor.layers.quantization.utils import marlin_utils
|
||||
from vllm.v1.worker import workspace as workspace_module
|
||||
|
||||
monkeypatch.setattr(workspace_module, "_manager", None)
|
||||
monkeypatch.setattr(marlin_utils, "num_compute_units", lambda _: 4)
|
||||
first = marlin_utils.get_marlin_workspace(torch.device("cpu"))
|
||||
second = marlin_utils.get_marlin_workspace(torch.device("cpu"))
|
||||
assert first.data_ptr() != second.data_ptr()
|
||||
assert first.shape == (4 * marlin_utils.MARLIN_MAX_BLOCKS_PER_SM,)
|
||||
assert torch.count_nonzero(first) == 0
|
||||
|
||||
|
||||
def test_model_cleanup(dist_init, default_vllm_config):
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import cast
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -124,3 +125,65 @@ def test_workspace_lane_validation(monkeypatch) -> None:
|
||||
|
||||
with pytest.raises(ValueError, match="at least one"):
|
||||
workspace.WorkspaceManager(torch.device("cpu"), num_lanes=0)
|
||||
|
||||
|
||||
def test_persistent_resources_are_lazy_and_isolated(monkeypatch) -> None:
|
||||
"""Only the requesting slot allocates; each ubatch/lane owns its resource."""
|
||||
active_ubatch = 0
|
||||
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: active_ubatch)
|
||||
manager = workspace.WorkspaceManager(
|
||||
torch.device("cpu"), num_ubatches=2, num_lanes=2
|
||||
)
|
||||
factory = Mock(side_effect=lambda: {"buffer": torch.zeros(8)})
|
||||
resources = []
|
||||
for ubatch in range(2):
|
||||
active_ubatch = ubatch
|
||||
for lane in range(2):
|
||||
with workspace.use_workspace_lane(lane):
|
||||
resource = manager.get_persistent_resource("scratch", factory)
|
||||
resources.append(resource)
|
||||
resource["buffer"].fill_(len(resources))
|
||||
assert manager.get_persistent_resource("scratch", factory) is resource
|
||||
assert factory.call_count == len(resources)
|
||||
|
||||
assert len({r["buffer"].data_ptr() for r in resources}) == 4
|
||||
for i, resource in enumerate(resources, 1):
|
||||
assert torch.all(resource["buffer"] == i)
|
||||
|
||||
|
||||
def test_persistent_resource_lock_applies_to_each_slot(monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0)
|
||||
manager = workspace.WorkspaceManager(torch.device("cpu"), num_lanes=2)
|
||||
factory = Mock(side_effect=object)
|
||||
first = manager.get_persistent_resource("scratch", factory)
|
||||
manager.lock()
|
||||
assert manager.get_persistent_resource("scratch", factory) is first
|
||||
with pytest.raises(AssertionError, match="was not allocated during warmup"):
|
||||
manager.get_persistent_resource("new", factory)
|
||||
with workspace.use_workspace_lane(1):
|
||||
with pytest.raises(AssertionError, match="was not allocated during warmup"):
|
||||
manager.get_persistent_resource("scratch", factory)
|
||||
assert factory.call_count == 1
|
||||
manager.unlock()
|
||||
assert manager.get_persistent_resource("scratch", factory) is not first
|
||||
assert manager.get_persistent_resource("scratch", factory) is first
|
||||
|
||||
|
||||
def test_persistent_tensor_preserves_contents_and_rejects_changes() -> None:
|
||||
"""Initialization happens once and transient scratch cannot overwrite locks."""
|
||||
manager = workspace.WorkspaceManager(torch.device("cpu"))
|
||||
first = manager.get_persistent("locks", (8,), torch.int32, zero_init=True)
|
||||
assert torch.count_nonzero(first) == 0
|
||||
first.fill_(1)
|
||||
(scratch,) = manager.get_simultaneous(((1024,), torch.uint8))
|
||||
scratch.zero_()
|
||||
assert manager.get_persistent("locks", (8,), torch.int32, zero_init=True) is first
|
||||
assert torch.all(first == 1)
|
||||
with pytest.raises(ValueError, match="requested shape"):
|
||||
manager.get_persistent("locks", (16,), torch.int32)
|
||||
with pytest.raises(ValueError, match="requested shape"):
|
||||
manager.get_persistent("locks", (8,), torch.int64)
|
||||
manager.lock()
|
||||
assert manager.get_persistent("locks", (8,), torch.int32) is first
|
||||
with pytest.raises(AssertionError, match="was not allocated during warmup"):
|
||||
manager.get_persistent("new", (8,), torch.int32)
|
||||
|
||||
Reference in New Issue
Block a user