[Bugfix][MRV2] Isolate sleep-mode KV allocations (#53508)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -1752,59 +1752,3 @@ def test_mamba_cache_raises_when_max_num_seqs_exceeds_blocks():
|
||||
|
||||
with pytest.raises(ValueError, match="max_num_seqs"):
|
||||
runner.initialize_kv_cache(kv_cache_config)
|
||||
|
||||
|
||||
class TestInitFp8KvScalesHybridModels:
|
||||
"""Verify init_fp8_kv_scales handles heterogeneous kv_caches entries.
|
||||
|
||||
Hybrid models (Mamba, DeltaNet) store per-layer state as a list of tensors
|
||||
rather than a single tensor. init_fp8_kv_scales must iterate both forms.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _make_runner_stub(kv_caches):
|
||||
runner = Mock(spec=GPUModelRunner)
|
||||
runner.cache_config = SimpleNamespace(cache_dtype="fp8_e4m3")
|
||||
runner.kv_caches = kv_caches
|
||||
runner.compilation_config = SimpleNamespace(static_forward_context={})
|
||||
runner.init_fp8_kv_scales = GPUModelRunner.init_fp8_kv_scales.__get__(
|
||||
runner, GPUModelRunner
|
||||
)
|
||||
return runner
|
||||
|
||||
def test_zeroes_both_tensor_and_list_entries(self):
|
||||
single_tensor = torch.ones(4, 8)
|
||||
list_tensors = [torch.ones(2, 4), torch.ones(3, 6)]
|
||||
|
||||
runner = self._make_runner_stub([single_tensor, list_tensors])
|
||||
runner.init_fp8_kv_scales()
|
||||
|
||||
assert (single_tensor == 0).all()
|
||||
assert all((t == 0).all() for t in list_tensors)
|
||||
|
||||
def test_skips_none_entries(self):
|
||||
tensor = torch.ones(4, 8)
|
||||
runner = self._make_runner_stub([None, tensor, None])
|
||||
runner.init_fp8_kv_scales()
|
||||
|
||||
assert (tensor == 0).all()
|
||||
|
||||
def test_noop_when_kv_cache_not_quantized(self):
|
||||
tensor = torch.ones(4, 8)
|
||||
runner = self._make_runner_stub([tensor])
|
||||
runner.cache_config.cache_dtype = "auto"
|
||||
runner.init_fp8_kv_scales()
|
||||
|
||||
assert (tensor == 1).all()
|
||||
|
||||
def test_mixed_none_tensor_and_list(self):
|
||||
t1 = torch.ones(2, 2)
|
||||
t2 = torch.ones(3, 3)
|
||||
list_entry = [torch.ones(1, 1), torch.ones(1, 1)]
|
||||
|
||||
runner = self._make_runner_stub([None, t1, list_entry, None, t2])
|
||||
runner.init_fp8_kv_scales()
|
||||
|
||||
assert (t1 == 0).all()
|
||||
assert (t2 == 0).all()
|
||||
assert all((t == 0).all() for t in list_entry)
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import AbstractContextManager
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.v1.worker.gpu.attn_utils as attn_utils
|
||||
import vllm.v1.worker.gpu_model_runner as gpu_model_runner
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
|
||||
class _AllocationScope(AbstractContextManager):
|
||||
def __init__(self) -> None:
|
||||
self.active = False
|
||||
|
||||
def __enter__(self):
|
||||
assert not self.active
|
||||
self.active = True
|
||||
return self
|
||||
|
||||
def __exit__(self, *args: Any) -> None:
|
||||
assert self.active
|
||||
self.active = False
|
||||
|
||||
|
||||
def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch) -> None:
|
||||
scope = _AllocationScope()
|
||||
kv_caches = {"layer": torch.empty(0)}
|
||||
|
||||
def allocate(*args, **kwargs):
|
||||
assert scope.active
|
||||
return kv_caches
|
||||
|
||||
def bind(*args, **kwargs):
|
||||
assert not scope.active
|
||||
|
||||
monkeypatch.setattr(attn_utils, "allocate_kv_cache", allocate)
|
||||
monkeypatch.setattr(attn_utils, "bind_kv_cache", bind)
|
||||
monkeypatch.setattr(attn_utils, "get_shared_kv_cache_layers", lambda config: {})
|
||||
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(get_resolved_kv_cache_layout=lambda: None),
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type="test")),
|
||||
)
|
||||
result = attn_utils.init_kv_cache(
|
||||
[],
|
||||
{},
|
||||
object(),
|
||||
torch.device("cpu"),
|
||||
[],
|
||||
config,
|
||||
kv_cache_allocation_context=scope,
|
||||
)
|
||||
|
||||
assert result is kv_caches
|
||||
assert not scope.active
|
||||
|
||||
|
||||
def test_mrv1_kv_pool_only_wraps_backing_allocation(monkeypatch) -> None:
|
||||
scope = _AllocationScope()
|
||||
kv_caches = {"layer": torch.empty(0)}
|
||||
|
||||
def allocate(*args, **kwargs):
|
||||
assert scope.active
|
||||
return kv_caches
|
||||
|
||||
def bind(*args, **kwargs):
|
||||
assert not scope.active
|
||||
|
||||
monkeypatch.setattr(gpu_model_runner, "allocate_kv_cache", allocate)
|
||||
monkeypatch.setattr(gpu_model_runner, "bind_kv_cache", bind)
|
||||
|
||||
runner = SimpleNamespace(
|
||||
device=torch.device("cpu"),
|
||||
cache_config=SimpleNamespace(get_resolved_kv_cache_layout=lambda: None),
|
||||
shared_kv_cache_layers={},
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type="test")),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
kv_caches=[],
|
||||
)
|
||||
result = gpu_model_runner.GPUModelRunner.initialize_kv_cache_tensors(
|
||||
runner,
|
||||
object(),
|
||||
[],
|
||||
kv_cache_allocation_context=scope,
|
||||
)
|
||||
|
||||
assert result is kv_caches
|
||||
assert not scope.active
|
||||
|
||||
|
||||
def test_kv_wake_does_not_run_model_runner_recovery() -> None:
|
||||
model = torch.nn.Module()
|
||||
model.register_buffer("_k_scale", torch.tensor(0.5))
|
||||
model.register_buffer("_v_scale", torch.tensor(0.25))
|
||||
|
||||
class Runner:
|
||||
def __init__(self) -> None:
|
||||
self.model = model
|
||||
self.layout_tensors = tuple(torch.tensor([i]) for i in range(5))
|
||||
self.recovery_calls = 0
|
||||
|
||||
def post_kv_cache_wake_up(self) -> None:
|
||||
self.recovery_calls += 1
|
||||
self.model.get_buffer("_k_scale").fill_(1.0)
|
||||
self.model.get_buffer("_v_scale").fill_(1.0)
|
||||
self.layout_tensors = tuple(torch.tensor([i]) for i in range(5))
|
||||
|
||||
runner = Runner()
|
||||
worker = cast(
|
||||
Worker,
|
||||
SimpleNamespace(
|
||||
_get_sleep_mode_backend=lambda: SimpleNamespace(resume=lambda tags: None),
|
||||
_sleep_saved_buffers={},
|
||||
_sleep_saved_draft_buffers={},
|
||||
model_runner=runner,
|
||||
synchronize_device=lambda: None,
|
||||
),
|
||||
)
|
||||
layout_tensors = runner.layout_tensors
|
||||
layout_ptrs = tuple(t.data_ptr() for t in layout_tensors)
|
||||
|
||||
Worker.wake_up(worker, tags=["kv_cache"])
|
||||
|
||||
assert runner.recovery_calls == 0
|
||||
assert model.get_buffer("_k_scale").item() == 0.5
|
||||
assert model.get_buffer("_v_scale").item() == 0.25
|
||||
assert all(
|
||||
actual is expected
|
||||
for actual, expected in zip(runner.layout_tensors, layout_tensors, strict=True)
|
||||
)
|
||||
assert tuple(t.data_ptr() for t in runner.layout_tensors) == layout_ptrs
|
||||
Reference in New Issue
Block a user