[Bugfix][V1] Free out-of-window blocks on the processed-token basis under async scheduling (#47728)
Signed-off-by: Saddss <28726669061@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Saddss <28726669061@qq.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Yifan Qiao
Saddss
Nick Hill
parent
934eeaecfb
commit
cd0de48d08
@@ -39,13 +39,18 @@ from vllm.v1.worker.mamba_utils import get_mamba_groups
|
||||
class StepAction:
|
||||
num_computed_tokens_start: int
|
||||
num_scheduled_tokens: int
|
||||
kv_cache_block_ids: list[int] # [] to follow last step
|
||||
kv_cache_block_ids: list[int] # per-block mask: 1=held, 0=freed/nulled
|
||||
preprocess_copy_idx: tuple[int, int] # -1, -1 for no copy
|
||||
postprocess_copy_idx: tuple[int, int] # -1, -1 for no copy
|
||||
|
||||
|
||||
num_speculative_tokens = 3
|
||||
|
||||
# Whether the run under test uses async scheduling. Set by each test entrypoint
|
||||
# before generation; consulted where the scheduler's optimistic token count must
|
||||
# be corrected for in-flight (possibly-rejected) speculative tokens.
|
||||
async_scheduling_mode = False
|
||||
|
||||
num_accepted_tokens = 1
|
||||
prompt_token_ids: list[int] = []
|
||||
MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8"
|
||||
@@ -249,7 +254,8 @@ def get_fake_execute_model_fn(original_execute_model_fn: Callable):
|
||||
scheduler_output.scheduled_cached_reqs.num_computed_tokens[0]
|
||||
)
|
||||
if (
|
||||
self.num_spec_tokens
|
||||
async_scheduling_mode
|
||||
and self.num_spec_tokens
|
||||
and num_prompt_tokens is not None
|
||||
and num_computed_tokens > num_prompt_tokens
|
||||
):
|
||||
@@ -496,7 +502,10 @@ def apply_patch(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(mamba_utils, "do_mamba_copy_block", fake_copy_fn)
|
||||
|
||||
|
||||
def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
def get_mamba_prefix_cache_step_configs(
|
||||
async_scheduling: bool = False,
|
||||
) -> dict[str, TestConfig]:
|
||||
a = async_scheduling
|
||||
tests = {
|
||||
"accept_1": TestConfig(
|
||||
num_prompt_tokens=554,
|
||||
@@ -504,14 +513,24 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=1,
|
||||
step_actions=[
|
||||
StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(556, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(557, 4, [], (0, 1), (-1, -1)),
|
||||
StepAction(558, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(559, 4, [], (-1, -1), (1, 0)),
|
||||
StepAction(560, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(561, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
555, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1)
|
||||
),
|
||||
StepAction(
|
||||
556, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1)
|
||||
),
|
||||
StepAction(557, 4, [1, 1, 1, 1, 1], (0, 1), (-1, -1)),
|
||||
StepAction(558, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(559, 4, [1, 1, 1, 1, 1], (-1, -1), (1, 0)),
|
||||
StepAction(560, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
561,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
# test case 2.1: no hit, accept 2 tokens
|
||||
@@ -521,11 +540,19 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=2,
|
||||
step_actions=[
|
||||
StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(556, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(558, 4, [], (1, 1), (2, 0)),
|
||||
StepAction(560, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(562, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
556, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1)
|
||||
),
|
||||
StepAction(558, 4, [1, 1, 1, 1, 1], (1, 1), (2, 0)),
|
||||
StepAction(560, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
562,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
# test case 2.2: no hit, accept 2 tokens
|
||||
@@ -535,10 +562,16 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=2,
|
||||
step_actions=[
|
||||
StepAction(0, 555, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(557, 4, [1, 1, 1, 1, 1], (1, 1), (-1, -1)),
|
||||
StepAction(559, 4, [], (-1, -1), (1, 0)),
|
||||
StepAction(561, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(559, 4, [1, 1, 1, 1, 1], (-1, -1), (1, 0)),
|
||||
StepAction(
|
||||
561,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(563, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
@@ -548,10 +581,18 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=3,
|
||||
step_actions=[
|
||||
StepAction(0, 553, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(553, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(556, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(559, 4, [], (2, 1), (1, 0)),
|
||||
StepAction(562, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(553, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
556, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1)
|
||||
),
|
||||
StepAction(559, 4, [1, 1, 1, 1, 1], (2, 1), (1, 0)),
|
||||
StepAction(
|
||||
562,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(565, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
@@ -561,10 +602,16 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=3,
|
||||
step_actions=[
|
||||
StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(557, 4, [1, 1, 1, 1, 1], (2, 1), (3, 0)),
|
||||
StepAction(560, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(563, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(560, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
563,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"accept_3_3": TestConfig(
|
||||
@@ -573,9 +620,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=3,
|
||||
step_actions=[
|
||||
StepAction(0, 555, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(558, 4, [1, 1, 1, 1, 1], (2, 1), (2, 0)),
|
||||
StepAction(561, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
561,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(564, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
@@ -585,9 +638,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, 553, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(553, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(553, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(557, 4, [1, 1, 1, 1, 1], (3, 1), (3, 0)),
|
||||
StepAction(561, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
561,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(565, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
@@ -597,9 +656,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(558, 4, [1, 1, 1, 1, 1], (3, 1), (2, 0)),
|
||||
StepAction(562, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
562,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(566, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
@@ -609,9 +674,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, 555, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(559, 4, [1, 1, 1, 1, 1], (3, 1), (1, 0)),
|
||||
StepAction(563, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
563,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(567, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
@@ -621,9 +692,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, 556, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(556, 4, [], (-1, -1), (3, 0)),
|
||||
StepAction(556, 4, [1, 1, 1, 1], (-1, -1), (3, 0)),
|
||||
StepAction(560, 4, [1, 1, 1, 1, 1], (0, 1), (-1, -1)),
|
||||
StepAction(564, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
564,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"prompt_block_size": TestConfig(
|
||||
@@ -642,7 +719,13 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
step_actions=[
|
||||
StepAction(0, 560, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(560, 560, [1, 1, 1, 1, 1], (0, 1), (-1, -1)),
|
||||
StepAction(560 * 2, 4, [0, 1, 1, 1, 1, 1], (1, 2), (-1, -1)),
|
||||
StepAction(
|
||||
560 * 2,
|
||||
4,
|
||||
[1, 1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1, 1],
|
||||
(1, 2),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"prompt_2_block_size_10": TestConfig(
|
||||
@@ -652,7 +735,13 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
step_actions=[
|
||||
StepAction(0, 560, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(560, 570, [1, 0, 1, 1, 1, 1], (0, 2), (-1, -1)),
|
||||
StepAction(560 * 2 + 10, 4, [0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
560 * 2 + 10,
|
||||
4,
|
||||
[1, 0, 1, 1, 1, 1] if a else [0, 0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"prompt_3_block_size": TestConfig(
|
||||
@@ -662,7 +751,13 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
step_actions=[
|
||||
StepAction(0, 560 * 2, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(560 * 2, 560, [0, 1, 1, 1, 1, 1], (1, 2), (-1, -1)),
|
||||
StepAction(560 * 3, 4, [0, 0, 1, 1, 1, 1, 1], (2, 3), (-1, -1)),
|
||||
StepAction(
|
||||
560 * 3,
|
||||
4,
|
||||
[0, 1, 1, 1, 1, 1, 1] if a else [0, 0, 1, 1, 1, 1, 1],
|
||||
(2, 3),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"prompt_3_block_size_10": TestConfig(
|
||||
@@ -672,7 +767,13 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
step_actions=[
|
||||
StepAction(0, 560 * 2, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(560 * 2, 570, [0, 1, 0, 1, 1, 1, 1], (1, 3), (-1, -1)),
|
||||
StepAction(560 * 3 + 10, 4, [0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
560 * 3 + 10,
|
||||
4,
|
||||
[0, 1, 0, 1, 1, 1, 1] if a else [0, 0, 0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"prompt_10_block_size": TestConfig(
|
||||
@@ -691,14 +792,18 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
StepAction(
|
||||
560 * 9,
|
||||
560,
|
||||
[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1]
|
||||
if a
|
||||
else [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
(8, 9),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(
|
||||
560 * 10,
|
||||
4,
|
||||
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1]
|
||||
if a
|
||||
else [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
(9, 10),
|
||||
(-1, -1),
|
||||
),
|
||||
@@ -720,34 +825,28 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
StepAction(
|
||||
560 * 9,
|
||||
560 + 10,
|
||||
[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1],
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1]
|
||||
if a
|
||||
else [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1],
|
||||
(8, 10),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
}
|
||||
|
||||
return tests
|
||||
|
||||
|
||||
def fill_following_kv_cache_block_ids(test_config: TestConfig) -> None:
|
||||
for step_action_prev, step_action_next in zip(
|
||||
test_config.step_actions[:-1], test_config.step_actions[1:]
|
||||
):
|
||||
if len(step_action_next.kv_cache_block_ids) == 0:
|
||||
step_action_next.kv_cache_block_ids = (
|
||||
step_action_prev.kv_cache_block_ids.copy()
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
def _run_mamba_prefix_cache_mrv1(
|
||||
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
|
||||
):
|
||||
global async_scheduling_mode
|
||||
async_scheduling_mode = async_scheduling
|
||||
run_ref_mamba_state_in_subprocess()
|
||||
apply_patch(monkeypatch)
|
||||
prompt_dataset = datasets.load_dataset("heheda/a_long_article")
|
||||
full_prompt = prompt_dataset["train"][0]["text"]
|
||||
tests = get_mamba_prefix_cache_step_configs()
|
||||
tests = get_mamba_prefix_cache_step_configs(async_scheduling)
|
||||
|
||||
engine = LLM(
|
||||
model=MODEL,
|
||||
@@ -761,6 +860,7 @@ def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
},
|
||||
max_num_batched_tokens=3072,
|
||||
hf_overrides={"num_hidden_layers": NUM_HIDDEN_LAYERS},
|
||||
async_scheduling=async_scheduling,
|
||||
seed=42,
|
||||
)
|
||||
global prompt_token_ids
|
||||
@@ -777,7 +877,6 @@ def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
)
|
||||
global cur_step_action_idx
|
||||
cur_step_action_idx = 0
|
||||
fill_following_kv_cache_block_ids(test_config)
|
||||
global step_actions
|
||||
step_actions = test_config.step_actions
|
||||
_ = engine.generate(
|
||||
@@ -800,7 +899,20 @@ def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=False)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=True)
|
||||
|
||||
|
||||
def _run_mamba_prefix_cache_mrv2(
|
||||
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
|
||||
):
|
||||
global async_scheduling_mode
|
||||
async_scheduling_mode = async_scheduling
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
envs.disable_envs_cache()
|
||||
@@ -998,11 +1110,12 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
max_num_batched_tokens=3072,
|
||||
max_model_len=BLOCK_SIZE * 12,
|
||||
hf_overrides={"num_hidden_layers": NUM_HIDDEN_LAYERS},
|
||||
async_scheduling=async_scheduling,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
try:
|
||||
tests = get_mamba_prefix_cache_step_configs()
|
||||
tests = get_mamba_prefix_cache_step_configs(async_scheduling)
|
||||
|
||||
global step_actions
|
||||
global cur_step_action_idx
|
||||
@@ -1010,7 +1123,6 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
for test_name, test_config in tests.items():
|
||||
num_accepted_tokens = test_config.num_accepted_tokens
|
||||
cur_step_action_idx = 0
|
||||
fill_following_kv_cache_block_ids(test_config)
|
||||
step_actions = test_config.step_actions
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
@@ -1053,3 +1165,13 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
del engine
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv2(monkeypatch, async_scheduling=False)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv2_async(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv2(monkeypatch, async_scheduling=True)
|
||||
|
||||
Reference in New Issue
Block a user