[Bugfix] Fix /wake_up crash on hybrid models (Mamba/DeltaNet) (#41602)

Signed-off-by: Kevin Glynn <kevglynn@gmail.com>
This commit is contained in:
Kevin Glynn
2026-07-29 00:17:35 +00:00
committed by GitHub
parent fe65aa6a97
commit 56f31af62a
2 changed files with 66 additions and 3 deletions
+56
View File
@@ -1669,3 +1669,59 @@ def test_mamba_cache_raises_when_max_num_seqs_exceeds_blocks():
with pytest.raises(ValueError, match="max_num_seqs"):
runner.initialize_kv_cache(kv_cache_config)
class TestInitFp8KvScalesHybridModels:
"""Verify init_fp8_kv_scales handles heterogeneous kv_caches entries.
Hybrid models (Mamba, DeltaNet) store per-layer state as a list of tensors
rather than a single tensor. init_fp8_kv_scales must iterate both forms.
"""
@staticmethod
def _make_runner_stub(kv_caches):
runner = Mock(spec=GPUModelRunner)
runner.cache_config = SimpleNamespace(cache_dtype="fp8_e4m3")
runner.kv_caches = kv_caches
runner.compilation_config = SimpleNamespace(static_forward_context={})
runner.init_fp8_kv_scales = GPUModelRunner.init_fp8_kv_scales.__get__(
runner, GPUModelRunner
)
return runner
def test_zeroes_both_tensor_and_list_entries(self):
single_tensor = torch.ones(4, 8)
list_tensors = [torch.ones(2, 4), torch.ones(3, 6)]
runner = self._make_runner_stub([single_tensor, list_tensors])
runner.init_fp8_kv_scales()
assert (single_tensor == 0).all()
assert all((t == 0).all() for t in list_tensors)
def test_skips_none_entries(self):
tensor = torch.ones(4, 8)
runner = self._make_runner_stub([None, tensor, None])
runner.init_fp8_kv_scales()
assert (tensor == 0).all()
def test_noop_when_kv_cache_not_quantized(self):
tensor = torch.ones(4, 8)
runner = self._make_runner_stub([tensor])
runner.cache_config.cache_dtype = "auto"
runner.init_fp8_kv_scales()
assert (tensor == 1).all()
def test_mixed_none_tensor_and_list(self):
t1 = torch.ones(2, 2)
t2 = torch.ones(3, 3)
list_entry = [torch.ones(1, 1), torch.ones(1, 1)]
runner = self._make_runner_stub([None, t1, list_entry, None, t2])
runner.init_fp8_kv_scales()
assert (t1 == 0).all()
assert (t2 == 0).all()
assert all((t == 0).all() for t in list_entry)
+10 -3
View File
@@ -991,9 +991,16 @@ class GPUModelRunner(
return
kv_caches = getattr(self, "kv_caches", [])
for cache_tensor in kv_caches:
if cache_tensor is not None:
cache_tensor.zero_()
for cache_entry in kv_caches:
if cache_entry is None:
continue
# Hybrid models (Mamba, DeltaNet) store per-layer state as a
# list of tensors rather than a single tensor.
if isinstance(cache_entry, list):
for t in cache_entry:
t.zero_()
else:
cache_entry.zero_()
k_attr_names = ("_k_scale", "k_scale")
v_attr_names = ("_v_scale", "v_scale")