[Bugfix][V1] Free out-of-window blocks on the processed-token basis under async scheduling (#47728)

Signed-off-by: Saddss <28726669061@qq.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Saddss <28726669061@qq.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Saddss
2026-07-08 13:34:19 +01:00
committed by GitHub
co-authored by Yifan Qiao Saddss Nick Hill
parent 934eeaecfb
commit cd0de48d08
14 changed files with 549 additions and 143 deletions
+183 -61
View File
@@ -39,13 +39,18 @@ from vllm.v1.worker.mamba_utils import get_mamba_groups
class StepAction:
num_computed_tokens_start: int
num_scheduled_tokens: int
kv_cache_block_ids: list[int] # [] to follow last step
kv_cache_block_ids: list[int] # per-block mask: 1=held, 0=freed/nulled
preprocess_copy_idx: tuple[int, int] # -1, -1 for no copy
postprocess_copy_idx: tuple[int, int] # -1, -1 for no copy
num_speculative_tokens = 3
# Whether the run under test uses async scheduling. Set by each test entrypoint
# before generation; consulted where the scheduler's optimistic token count must
# be corrected for in-flight (possibly-rejected) speculative tokens.
async_scheduling_mode = False
num_accepted_tokens = 1
prompt_token_ids: list[int] = []
MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8"
@@ -249,7 +254,8 @@ def get_fake_execute_model_fn(original_execute_model_fn: Callable):
scheduler_output.scheduled_cached_reqs.num_computed_tokens[0]
)
if (
self.num_spec_tokens
async_scheduling_mode
and self.num_spec_tokens
and num_prompt_tokens is not None
and num_computed_tokens > num_prompt_tokens
):
@@ -496,7 +502,10 @@ def apply_patch(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(mamba_utils, "do_mamba_copy_block", fake_copy_fn)
def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
def get_mamba_prefix_cache_step_configs(
async_scheduling: bool = False,
) -> dict[str, TestConfig]:
a = async_scheduling
tests = {
"accept_1": TestConfig(
num_prompt_tokens=554,
@@ -504,14 +513,24 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
num_accepted_tokens=1,
step_actions=[
StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(554, 4, [], (-1, -1), (-1, -1)),
StepAction(555, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(556, 4, [], (-1, -1), (-1, -1)),
StepAction(557, 4, [], (0, 1), (-1, -1)),
StepAction(558, 4, [], (-1, -1), (-1, -1)),
StepAction(559, 4, [], (-1, -1), (1, 0)),
StepAction(560, 4, [], (-1, -1), (-1, -1)),
StepAction(561, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
555, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1)
),
StepAction(
556, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1)
),
StepAction(557, 4, [1, 1, 1, 1, 1], (0, 1), (-1, -1)),
StepAction(558, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(559, 4, [1, 1, 1, 1, 1], (-1, -1), (1, 0)),
StepAction(560, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
561,
4,
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
(-1, -1),
(-1, -1),
),
],
),
# test case 2.1: no hit, accept 2 tokens
@@ -521,11 +540,19 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
num_accepted_tokens=2,
step_actions=[
StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(554, 4, [], (-1, -1), (-1, -1)),
StepAction(556, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(558, 4, [], (1, 1), (2, 0)),
StepAction(560, 4, [], (-1, -1), (-1, -1)),
StepAction(562, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
556, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1)
),
StepAction(558, 4, [1, 1, 1, 1, 1], (1, 1), (2, 0)),
StepAction(560, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
562,
4,
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
(-1, -1),
(-1, -1),
),
],
),
# test case 2.2: no hit, accept 2 tokens
@@ -535,10 +562,16 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
num_accepted_tokens=2,
step_actions=[
StepAction(0, 555, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(555, 4, [], (-1, -1), (-1, -1)),
StepAction(555, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(557, 4, [1, 1, 1, 1, 1], (1, 1), (-1, -1)),
StepAction(559, 4, [], (-1, -1), (1, 0)),
StepAction(561, 4, [], (-1, -1), (-1, -1)),
StepAction(559, 4, [1, 1, 1, 1, 1], (-1, -1), (1, 0)),
StepAction(
561,
4,
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
(-1, -1),
(-1, -1),
),
StepAction(563, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
],
),
@@ -548,10 +581,18 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
num_accepted_tokens=3,
step_actions=[
StepAction(0, 553, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(553, 4, [], (-1, -1), (-1, -1)),
StepAction(556, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(559, 4, [], (2, 1), (1, 0)),
StepAction(562, 4, [], (-1, -1), (-1, -1)),
StepAction(553, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
556, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1)
),
StepAction(559, 4, [1, 1, 1, 1, 1], (2, 1), (1, 0)),
StepAction(
562,
4,
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
(-1, -1),
(-1, -1),
),
StepAction(565, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
],
),
@@ -561,10 +602,16 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
num_accepted_tokens=3,
step_actions=[
StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(554, 4, [], (-1, -1), (-1, -1)),
StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(557, 4, [1, 1, 1, 1, 1], (2, 1), (3, 0)),
StepAction(560, 4, [], (-1, -1), (-1, -1)),
StepAction(563, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(560, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
563,
4,
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
(-1, -1),
(-1, -1),
),
],
),
"accept_3_3": TestConfig(
@@ -573,9 +620,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
num_accepted_tokens=3,
step_actions=[
StepAction(0, 555, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(555, 4, [], (-1, -1), (-1, -1)),
StepAction(555, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(558, 4, [1, 1, 1, 1, 1], (2, 1), (2, 0)),
StepAction(561, 4, [], (-1, -1), (-1, -1)),
StepAction(
561,
4,
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
(-1, -1),
(-1, -1),
),
StepAction(564, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
],
),
@@ -585,9 +638,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
num_accepted_tokens=4,
step_actions=[
StepAction(0, 553, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(553, 4, [], (-1, -1), (-1, -1)),
StepAction(553, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(557, 4, [1, 1, 1, 1, 1], (3, 1), (3, 0)),
StepAction(561, 4, [], (-1, -1), (-1, -1)),
StepAction(
561,
4,
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
(-1, -1),
(-1, -1),
),
StepAction(565, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
],
),
@@ -597,9 +656,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
num_accepted_tokens=4,
step_actions=[
StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(554, 4, [], (-1, -1), (-1, -1)),
StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(558, 4, [1, 1, 1, 1, 1], (3, 1), (2, 0)),
StepAction(562, 4, [], (-1, -1), (-1, -1)),
StepAction(
562,
4,
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
(-1, -1),
(-1, -1),
),
StepAction(566, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
],
),
@@ -609,9 +674,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
num_accepted_tokens=4,
step_actions=[
StepAction(0, 555, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(555, 4, [], (-1, -1), (-1, -1)),
StepAction(555, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(559, 4, [1, 1, 1, 1, 1], (3, 1), (1, 0)),
StepAction(563, 4, [], (-1, -1), (-1, -1)),
StepAction(
563,
4,
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
(-1, -1),
(-1, -1),
),
StepAction(567, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
],
),
@@ -621,9 +692,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
num_accepted_tokens=4,
step_actions=[
StepAction(0, 556, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(556, 4, [], (-1, -1), (3, 0)),
StepAction(556, 4, [1, 1, 1, 1], (-1, -1), (3, 0)),
StepAction(560, 4, [1, 1, 1, 1, 1], (0, 1), (-1, -1)),
StepAction(564, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
564,
4,
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
(-1, -1),
(-1, -1),
),
],
),
"prompt_block_size": TestConfig(
@@ -642,7 +719,13 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
step_actions=[
StepAction(0, 560, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(560, 560, [1, 1, 1, 1, 1], (0, 1), (-1, -1)),
StepAction(560 * 2, 4, [0, 1, 1, 1, 1, 1], (1, 2), (-1, -1)),
StepAction(
560 * 2,
4,
[1, 1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1, 1],
(1, 2),
(-1, -1),
),
],
),
"prompt_2_block_size_10": TestConfig(
@@ -652,7 +735,13 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
step_actions=[
StepAction(0, 560, [1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(560, 570, [1, 0, 1, 1, 1, 1], (0, 2), (-1, -1)),
StepAction(560 * 2 + 10, 4, [0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
560 * 2 + 10,
4,
[1, 0, 1, 1, 1, 1] if a else [0, 0, 1, 1, 1, 1],
(-1, -1),
(-1, -1),
),
],
),
"prompt_3_block_size": TestConfig(
@@ -662,7 +751,13 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
step_actions=[
StepAction(0, 560 * 2, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(560 * 2, 560, [0, 1, 1, 1, 1, 1], (1, 2), (-1, -1)),
StepAction(560 * 3, 4, [0, 0, 1, 1, 1, 1, 1], (2, 3), (-1, -1)),
StepAction(
560 * 3,
4,
[0, 1, 1, 1, 1, 1, 1] if a else [0, 0, 1, 1, 1, 1, 1],
(2, 3),
(-1, -1),
),
],
),
"prompt_3_block_size_10": TestConfig(
@@ -672,7 +767,13 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
step_actions=[
StepAction(0, 560 * 2, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(560 * 2, 570, [0, 1, 0, 1, 1, 1, 1], (1, 3), (-1, -1)),
StepAction(560 * 3 + 10, 4, [0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
560 * 3 + 10,
4,
[0, 1, 0, 1, 1, 1, 1] if a else [0, 0, 0, 1, 1, 1, 1],
(-1, -1),
(-1, -1),
),
],
),
"prompt_10_block_size": TestConfig(
@@ -691,14 +792,18 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
StepAction(
560 * 9,
560,
[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1]
if a
else [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
(8, 9),
(-1, -1),
),
StepAction(
560 * 10,
4,
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1]
if a
else [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
(9, 10),
(-1, -1),
),
@@ -720,34 +825,28 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
StepAction(
560 * 9,
560 + 10,
[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1],
[0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1]
if a
else [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1],
(8, 10),
(-1, -1),
),
],
),
}
return tests
def fill_following_kv_cache_block_ids(test_config: TestConfig) -> None:
for step_action_prev, step_action_next in zip(
test_config.step_actions[:-1], test_config.step_actions[1:]
):
if len(step_action_next.kv_cache_block_ids) == 0:
step_action_next.kv_cache_block_ids = (
step_action_prev.kv_cache_block_ids.copy()
)
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
def _run_mamba_prefix_cache_mrv1(
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
):
global async_scheduling_mode
async_scheduling_mode = async_scheduling
run_ref_mamba_state_in_subprocess()
apply_patch(monkeypatch)
prompt_dataset = datasets.load_dataset("heheda/a_long_article")
full_prompt = prompt_dataset["train"][0]["text"]
tests = get_mamba_prefix_cache_step_configs()
tests = get_mamba_prefix_cache_step_configs(async_scheduling)
engine = LLM(
model=MODEL,
@@ -761,6 +860,7 @@ def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
},
max_num_batched_tokens=3072,
hf_overrides={"num_hidden_layers": NUM_HIDDEN_LAYERS},
async_scheduling=async_scheduling,
seed=42,
)
global prompt_token_ids
@@ -777,7 +877,6 @@ def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
)
global cur_step_action_idx
cur_step_action_idx = 0
fill_following_kv_cache_block_ids(test_config)
global step_actions
step_actions = test_config.step_actions
_ = engine.generate(
@@ -800,7 +899,20 @@ def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=False)
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=True)
def _run_mamba_prefix_cache_mrv2(
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
):
global async_scheduling_mode
async_scheduling_mode = async_scheduling
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
envs.disable_envs_cache()
@@ -998,11 +1110,12 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
max_num_batched_tokens=3072,
max_model_len=BLOCK_SIZE * 12,
hf_overrides={"num_hidden_layers": NUM_HIDDEN_LAYERS},
async_scheduling=async_scheduling,
seed=42,
)
try:
tests = get_mamba_prefix_cache_step_configs()
tests = get_mamba_prefix_cache_step_configs(async_scheduling)
global step_actions
global cur_step_action_idx
@@ -1010,7 +1123,6 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
for test_name, test_config in tests.items():
num_accepted_tokens = test_config.num_accepted_tokens
cur_step_action_idx = 0
fill_following_kv_cache_block_ids(test_config)
step_actions = test_config.step_actions
sampling_params = SamplingParams(
temperature=0.0,
@@ -1053,3 +1165,13 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
del engine
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv2(monkeypatch, async_scheduling=False)
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv2_async(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv2(monkeypatch, async_scheduling=True)