[K3] support recoverssm for K3 (#51855)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jiangyun Zhu
2026-08-17 19:34:14 +08:00
committed by GitHub
co-authored by OpenAI Codex Benjamin Chislett mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f27ae25473
commit 70afdedc10
19 changed files with 2235 additions and 75 deletions
@@ -1,11 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.attention.backends.recoverssm_metadata import (
RecoverSSMMetadata,
RecoverSSMPostprocessMetadata,
)
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
from vllm.v1.worker.gpu.model_states.recoverssm import RecoverSSMState
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
@@ -18,6 +26,7 @@ def test_postprocess_state_scalar_with_int32_mapping(
(4,), 9, dtype=torch.int32, device="cuda"
)
state._align_mode = False
state.recoverssm = None
state._mamba_ctx = None
idx_mapping = torch.tensor([2, -1, 0], dtype=torch.int32, device="cuda")
@@ -27,3 +36,61 @@ def test_postprocess_state_scalar_with_int32_mapping(
[expected_value, 9, expected_value, 9], dtype=torch.int32, device="cuda"
)
torch.testing.assert_close(state.num_accepted_tokens_gpu, expected)
def test_recoverssm_commits_accepted_window_after_v2_sampling() -> None:
state = RecoverSSMState()
metadata = Mock(spec=RecoverSSMMetadata)
metadata.commit_recoverssm_state.return_value = None
num_sampled = torch.tensor([3, 1], dtype=torch.int32)
idx_mapping = torch.tensor([0, 1], dtype=torch.int32)
num_accepted_tokens = torch.ones(2, dtype=torch.int32)
group = SimpleNamespace(layer_names=["layer"])
state.record_step({"layer": metadata}, [[group]], for_capture=False)
state.commit_step(
num_sampled,
idx_mapping,
state_indices=None,
num_accepted_tokens=num_accepted_tokens,
)
state.commit_step(
num_sampled,
idx_mapping,
state_indices=None,
num_accepted_tokens=num_accepted_tokens,
)
metadata.commit_recoverssm_state.assert_called_once_with(num_sampled)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_recoverssm_align_tracks_mixed_batch_state_and_neutralizes_copy_bias() -> None:
state = object.__new__(MambaHybridModelState)
state._align_mode = True
state._mamba_ctx = None
state._mamba_state_idx_gpu = torch.full((5,), -1, dtype=torch.int32, device="cuda")
state.recoverssm = RecoverSSMState()
state.num_accepted_tokens_gpu = torch.full(
(5,), 9, dtype=torch.int32, device="cuda"
)
metadata = Mock(spec=RecoverSSMMetadata)
metadata.commit_recoverssm_state.return_value = RecoverSSMPostprocessMetadata(
num_spec_decodes=1,
request_indices=torch.tensor([1], dtype=torch.int32, device="cuda"),
num_computed_tokens=torch.tensor([6, 7], dtype=torch.int32, device="cuda"),
block_size=8,
block_table=torch.zeros((2, 4), dtype=torch.int32, device="cuda"),
)
num_sampled = torch.tensor([2, 3], dtype=torch.int32, device="cuda")
idx_mapping = torch.tensor([3, 1], dtype=torch.int32, device="cuda")
group = SimpleNamespace(layer_names=["layer"])
state.recoverssm.record_step({"layer": metadata}, [[group]], for_capture=False)
state.postprocess_state(idx_mapping, num_sampled)
expected_state_indices = [-1, 1, -1, -1, -1]
assert state._mamba_state_idx_gpu.tolist() == expected_state_indices
expected_accepted = [9, 1, 9, 2, 9]
assert state.num_accepted_tokens_gpu.tolist() == expected_accepted
+36
View File
@@ -490,6 +490,42 @@ def test_stage_postprocess_inputs_to_gpu_asserts_on_missing_state_idx():
)
def test_gpu_context_ignores_auxiliary_cache_tensors() -> None:
device = torch.device("cpu")
config = _TestConfig(num_layers=1)
layer_names = ["layer_0"]
kv_cache_config = _make_kv_cache_config(config, layer_names)
conv_state = torch.empty(
config.num_blocks,
config.conv_width,
config.conv_inner_dim,
dtype=config.dtype,
)
temporal_state = torch.empty(
config.num_blocks, config.temporal_state_dim, dtype=config.dtype
)
attention = MagicMock()
attention.kv_cache = [
conv_state,
temporal_state,
*(torch.empty(config.num_blocks, 1) for _ in range(4)),
]
context = _make_gpu_ctx(config, kv_cache_config, device)
context.initialize_from_forward_context(
kv_cache_config,
{"layer_0": attention},
_COPY_FUNCS,
[torch.zeros(1, 1, dtype=torch.int32)],
)
assert context.is_initialized
assert context.state_base_addrs.tolist() == [
conv_state.data_ptr(),
temporal_state.data_ptr(),
]
def _run_gpu_postprocess(
gpu_ctx: MambaSpecDecodeGPUContext,
*,