From 56f31af62afe6553369b14af225bfafb788e99a0 Mon Sep 17 00:00:00 2001 From: Kevin Glynn Date: Tue, 28 Jul 2026 20:17:35 -0400 Subject: [PATCH] [Bugfix] Fix /wake_up crash on hybrid models (Mamba/DeltaNet) (#41602) Signed-off-by: Kevin Glynn --- tests/v1/worker/test_gpu_model_runner.py | 56 ++++++++++++++++++++++++ vllm/v1/worker/gpu_model_runner.py | 13 ++++-- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 79e3a60e98..d2d89a74fc 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -1669,3 +1669,59 @@ def test_mamba_cache_raises_when_max_num_seqs_exceeds_blocks(): with pytest.raises(ValueError, match="max_num_seqs"): runner.initialize_kv_cache(kv_cache_config) + + +class TestInitFp8KvScalesHybridModels: + """Verify init_fp8_kv_scales handles heterogeneous kv_caches entries. + + Hybrid models (Mamba, DeltaNet) store per-layer state as a list of tensors + rather than a single tensor. init_fp8_kv_scales must iterate both forms. + """ + + @staticmethod + def _make_runner_stub(kv_caches): + runner = Mock(spec=GPUModelRunner) + runner.cache_config = SimpleNamespace(cache_dtype="fp8_e4m3") + runner.kv_caches = kv_caches + runner.compilation_config = SimpleNamespace(static_forward_context={}) + runner.init_fp8_kv_scales = GPUModelRunner.init_fp8_kv_scales.__get__( + runner, GPUModelRunner + ) + return runner + + def test_zeroes_both_tensor_and_list_entries(self): + single_tensor = torch.ones(4, 8) + list_tensors = [torch.ones(2, 4), torch.ones(3, 6)] + + runner = self._make_runner_stub([single_tensor, list_tensors]) + runner.init_fp8_kv_scales() + + assert (single_tensor == 0).all() + assert all((t == 0).all() for t in list_tensors) + + def test_skips_none_entries(self): + tensor = torch.ones(4, 8) + runner = self._make_runner_stub([None, tensor, None]) + runner.init_fp8_kv_scales() + + assert (tensor == 0).all() + + def test_noop_when_kv_cache_not_quantized(self): + tensor = torch.ones(4, 8) + runner = self._make_runner_stub([tensor]) + runner.cache_config.cache_dtype = "auto" + runner.init_fp8_kv_scales() + + assert (tensor == 1).all() + + def test_mixed_none_tensor_and_list(self): + t1 = torch.ones(2, 2) + t2 = torch.ones(3, 3) + list_entry = [torch.ones(1, 1), torch.ones(1, 1)] + + runner = self._make_runner_stub([None, t1, list_entry, None, t2]) + runner.init_fp8_kv_scales() + + assert (t1 == 0).all() + assert (t2 == 0).all() + assert all((t == 0).all() for t in list_entry) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index dd31ddab67..0859b6dacd 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -991,9 +991,16 @@ class GPUModelRunner( return kv_caches = getattr(self, "kv_caches", []) - for cache_tensor in kv_caches: - if cache_tensor is not None: - cache_tensor.zero_() + for cache_entry in kv_caches: + if cache_entry is None: + continue + # Hybrid models (Mamba, DeltaNet) store per-layer state as a + # list of tensors rather than a single tensor. + if isinstance(cache_entry, list): + for t in cache_entry: + t.zero_() + else: + cache_entry.zero_() k_attr_names = ("_k_scale", "k_scale") v_attr_names = ("_v_scale", "v_scale")