[Bugfix][MRV2] Isolate sleep-mode KV allocations (#53508)

Signed-off-by: Ronald1995 <ronaldautomobile@163.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Ronald
2026-08-27 15:08:28 -07:00
committed by GitHub
co-authored by aoshen02
parent d262964e8c
commit 479eeb32d2
9 changed files with 228 additions and 146 deletions
-56
View File
@@ -1752,59 +1752,3 @@ def test_mamba_cache_raises_when_max_num_seqs_exceeds_blocks():
with pytest.raises(ValueError, match="max_num_seqs"):
runner.initialize_kv_cache(kv_cache_config)
class TestInitFp8KvScalesHybridModels:
"""Verify init_fp8_kv_scales handles heterogeneous kv_caches entries.
Hybrid models (Mamba, DeltaNet) store per-layer state as a list of tensors
rather than a single tensor. init_fp8_kv_scales must iterate both forms.
"""
@staticmethod
def _make_runner_stub(kv_caches):
runner = Mock(spec=GPUModelRunner)
runner.cache_config = SimpleNamespace(cache_dtype="fp8_e4m3")
runner.kv_caches = kv_caches
runner.compilation_config = SimpleNamespace(static_forward_context={})
runner.init_fp8_kv_scales = GPUModelRunner.init_fp8_kv_scales.__get__(
runner, GPUModelRunner
)
return runner
def test_zeroes_both_tensor_and_list_entries(self):
single_tensor = torch.ones(4, 8)
list_tensors = [torch.ones(2, 4), torch.ones(3, 6)]
runner = self._make_runner_stub([single_tensor, list_tensors])
runner.init_fp8_kv_scales()
assert (single_tensor == 0).all()
assert all((t == 0).all() for t in list_tensors)
def test_skips_none_entries(self):
tensor = torch.ones(4, 8)
runner = self._make_runner_stub([None, tensor, None])
runner.init_fp8_kv_scales()
assert (tensor == 0).all()
def test_noop_when_kv_cache_not_quantized(self):
tensor = torch.ones(4, 8)
runner = self._make_runner_stub([tensor])
runner.cache_config.cache_dtype = "auto"
runner.init_fp8_kv_scales()
assert (tensor == 1).all()
def test_mixed_none_tensor_and_list(self):
t1 = torch.ones(2, 2)
t2 = torch.ones(3, 3)
list_entry = [torch.ones(1, 1), torch.ones(1, 1)]
runner = self._make_runner_stub([None, t1, list_entry, None, t2])
runner.init_fp8_kv_scales()
assert (t1 == 0).all()
assert (t2 == 0).all()
assert all((t == 0).all() for t in list_entry)