[Bugfix][MRV2] Isolate sleep-mode KV allocations (#53508)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -1752,59 +1752,3 @@ def test_mamba_cache_raises_when_max_num_seqs_exceeds_blocks():
|
||||
|
||||
with pytest.raises(ValueError, match="max_num_seqs"):
|
||||
runner.initialize_kv_cache(kv_cache_config)
|
||||
|
||||
|
||||
class TestInitFp8KvScalesHybridModels:
|
||||
"""Verify init_fp8_kv_scales handles heterogeneous kv_caches entries.
|
||||
|
||||
Hybrid models (Mamba, DeltaNet) store per-layer state as a list of tensors
|
||||
rather than a single tensor. init_fp8_kv_scales must iterate both forms.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _make_runner_stub(kv_caches):
|
||||
runner = Mock(spec=GPUModelRunner)
|
||||
runner.cache_config = SimpleNamespace(cache_dtype="fp8_e4m3")
|
||||
runner.kv_caches = kv_caches
|
||||
runner.compilation_config = SimpleNamespace(static_forward_context={})
|
||||
runner.init_fp8_kv_scales = GPUModelRunner.init_fp8_kv_scales.__get__(
|
||||
runner, GPUModelRunner
|
||||
)
|
||||
return runner
|
||||
|
||||
def test_zeroes_both_tensor_and_list_entries(self):
|
||||
single_tensor = torch.ones(4, 8)
|
||||
list_tensors = [torch.ones(2, 4), torch.ones(3, 6)]
|
||||
|
||||
runner = self._make_runner_stub([single_tensor, list_tensors])
|
||||
runner.init_fp8_kv_scales()
|
||||
|
||||
assert (single_tensor == 0).all()
|
||||
assert all((t == 0).all() for t in list_tensors)
|
||||
|
||||
def test_skips_none_entries(self):
|
||||
tensor = torch.ones(4, 8)
|
||||
runner = self._make_runner_stub([None, tensor, None])
|
||||
runner.init_fp8_kv_scales()
|
||||
|
||||
assert (tensor == 0).all()
|
||||
|
||||
def test_noop_when_kv_cache_not_quantized(self):
|
||||
tensor = torch.ones(4, 8)
|
||||
runner = self._make_runner_stub([tensor])
|
||||
runner.cache_config.cache_dtype = "auto"
|
||||
runner.init_fp8_kv_scales()
|
||||
|
||||
assert (tensor == 1).all()
|
||||
|
||||
def test_mixed_none_tensor_and_list(self):
|
||||
t1 = torch.ones(2, 2)
|
||||
t2 = torch.ones(3, 3)
|
||||
list_entry = [torch.ones(1, 1), torch.ones(1, 1)]
|
||||
|
||||
runner = self._make_runner_stub([None, t1, list_entry, None, t2])
|
||||
runner.init_fp8_kv_scales()
|
||||
|
||||
assert (t1 == 0).all()
|
||||
assert (t2 == 0).all()
|
||||
assert all((t == 0).all() for t in list_entry)
|
||||
|
||||
Reference in New Issue
Block a user