From 07950d47347f7fff41f98616096fd10349171e3d Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:29:53 -0400 Subject: [PATCH] [Kimi Bug] Fix kimi k3 startup cuda graph issue with recoverSSM (#55774) Signed-off-by: yewentao256 --- tests/v1/cudagraph/test_cudagraph_manager.py | 11 ++++++++++- vllm/v1/worker/gpu/cudagraph_utils.py | 6 +++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/v1/cudagraph/test_cudagraph_manager.py b/tests/v1/cudagraph/test_cudagraph_manager.py index c6229548f0..c907be36b8 100644 --- a/tests/v1/cudagraph/test_cudagraph_manager.py +++ b/tests/v1/cudagraph/test_cudagraph_manager.py @@ -172,6 +172,8 @@ def _create_decode_vllm_config( capture_sizes: list[int], num_speculative_tokens: int = 0, dynamic_spec_schedule: list[tuple[int, int, int]] | None = None, + max_num_seqs: int = 8, + use_kda_recoverssm: bool = False, ) -> MagicMock: compilation_config = CompilationConfig( cudagraph_mode="FULL_AND_PIECEWISE", @@ -182,8 +184,11 @@ def _create_decode_vllm_config( vllm_config = MagicMock(spec=VllmConfig) vllm_config.compilation_config = compilation_config - vllm_config.scheduler_config = SchedulerConfig.default_factory(max_num_seqs=8) + vllm_config.scheduler_config = SchedulerConfig.default_factory( + max_num_seqs=max_num_seqs + ) vllm_config.parallel_config = ParallelConfig() + vllm_config.cache_config = SimpleNamespace(use_kda_recoverssm=use_kda_recoverssm) vllm_config.num_speculative_tokens = num_speculative_tokens if dynamic_spec_schedule is None: vllm_config.speculative_config = None @@ -201,6 +206,8 @@ def _make_spec_decode_manager( capture_sizes: list[int] | None = None, num_speculative_tokens: int = 0, dynamic_spec_schedule: list[tuple[int, int, int]] | None = None, + max_num_seqs: int = 8, + use_kda_recoverssm: bool = False, ) -> gpu_cudagraph_utils.CudaGraphManager: monkeypatch.setattr( gpu_cudagraph_utils, @@ -217,6 +224,8 @@ def _make_spec_decode_manager( capture_sizes or [1, 2, 4, 8, 16, 24], num_speculative_tokens=num_speculative_tokens, dynamic_spec_schedule=dynamic_spec_schedule, + max_num_seqs=max_num_seqs, + use_kda_recoverssm=use_kda_recoverssm, ), device=torch.device("cpu"), cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 66051a9fe2..4191f150ec 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -299,7 +299,11 @@ class CudaGraphManager: if desc not in descs_by_mode[decode_mode]: descs_by_mode[decode_mode].append(desc) - if mixed_mode: + # recoverSSM cannot capture a dummy query wider than its workspace. + if mixed_mode and ( + not self.vllm_config.cache_config.use_kda_recoverssm + or num_tokens <= max_decode_tokens + ): # for PIECEWISE graphs there is no limit on requests when replaying # i.e. no request padding is needed, so we leave it as None. # For breakable PW graphs, break-point kernels read the real batch