[Bugfix][V1] Free out-of-window blocks on the processed-token basis under async scheduling (#47728)
Signed-off-by: Saddss <28726669061@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Saddss <28726669061@qq.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Yifan Qiao
Saddss
Nick Hill
parent
934eeaecfb
commit
cd0de48d08
@@ -81,7 +81,7 @@ def assert_encoder_kv_cache_spec(engine: LLM) -> None:
|
||||
assert spec.sliding_window == cdiv(750, 4) + 1 == 189
|
||||
assert (
|
||||
spec.max_admission_blocks_per_request(
|
||||
max_num_batched_tokens=1,
|
||||
max_in_flight_tokens=1,
|
||||
max_model_len=vllm_config.model_config.max_model_len,
|
||||
)
|
||||
== 13
|
||||
|
||||
@@ -1404,12 +1404,15 @@ def test_get_max_concurrency_for_kv_cache_config():
|
||||
enable_chunked_prefill=True,
|
||||
max_model_len=model_config.max_model_len,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
# Pin to sync: SWA per-request bounds grow with overlapping batches.
|
||||
async_scheduling=False,
|
||||
)
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
scheduler_config=scheduler_config,
|
||||
)
|
||||
assert vllm_config.max_concurrent_batches == 1
|
||||
|
||||
full_attention_spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
|
||||
@@ -3453,7 +3453,8 @@ def test_can_fit_full_sequence_swa_cap_admits_long_prompt():
|
||||
manager = make_kv_cache_manager(
|
||||
config,
|
||||
max_model_len=max_model_len,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
# Single (sync) batch in flight, so in-flight tokens == batched tokens.
|
||||
max_in_flight_tokens=max_num_batched_tokens,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
@@ -3509,7 +3510,8 @@ def test_can_fit_full_sequence_full_attention_still_gates_oversized():
|
||||
manager = make_kv_cache_manager(
|
||||
config,
|
||||
max_model_len=max_model_len,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
# Single (sync) batch in flight, so in-flight tokens == batched tokens.
|
||||
max_in_flight_tokens=max_num_batched_tokens,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Out-of-window block frees vs in-flight GPU steps.
|
||||
|
||||
With async scheduling / PP, `num_computed_tokens` optimistically includes
|
||||
tokens of unprocessed steps whose attention windows still read the blocks
|
||||
just below the optimistic boundary (and rejected spec tokens can roll it
|
||||
back), so `allocate_slots` frees on the processed-token basis:
|
||||
`num_computed_tokens - num_in_flight_tokens`.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import ChunkedLocalAttentionSpec, SlidingWindowSpec
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
|
||||
from .utils import create_requests, create_scheduler, mock_kv
|
||||
|
||||
NUM_PROMPT_TOKENS = 100
|
||||
BLOCK_SIZE = 16
|
||||
SLIDING_WINDOW = 16
|
||||
# Tokens 0..84 are outside the window of the next token to compute
|
||||
# (100 - 16 + 1 = 85), i.e. 5 full blocks.
|
||||
NUM_OUT_OF_WINDOW_BLOCKS = 85 // BLOCK_SIZE
|
||||
# Chunked-local skips whole chunks left of the current one:
|
||||
# (100 // 32) * 32 = 96 settled tokens -> 6 full blocks.
|
||||
CHUNK_SIZE = 32
|
||||
NUM_OUT_OF_CHUNK_BLOCKS = (NUM_PROMPT_TOKENS // CHUNK_SIZE) * CHUNK_SIZE // BLOCK_SIZE
|
||||
|
||||
|
||||
def _make_model_runner_output(
|
||||
scheduler_output: SchedulerOutput,
|
||||
token_id: int = 0,
|
||||
) -> ModelRunnerOutput:
|
||||
req_ids = list(scheduler_output.num_scheduled_tokens.keys())
|
||||
return ModelRunnerOutput(
|
||||
req_ids=req_ids,
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(req_ids)},
|
||||
sampled_token_ids=[[token_id] for _ in req_ids],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
)
|
||||
|
||||
|
||||
def _create_swa_scheduler(async_scheduling: bool):
|
||||
return create_scheduler(
|
||||
block_size=BLOCK_SIZE,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_spec=SlidingWindowSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=SLIDING_WINDOW,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _create_chunked_scheduler(async_scheduling: bool):
|
||||
return create_scheduler(
|
||||
block_size=BLOCK_SIZE,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_spec=ChunkedLocalAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
attention_chunk_size=CHUNK_SIZE,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _num_null_blocks(scheduler, request_id: str) -> int:
|
||||
manager = scheduler.kv_cache_manager.coordinator.single_type_managers[0]
|
||||
null_block = manager._null_block
|
||||
return sum(1 for b in manager.req_to_blocks[request_id] if b is null_block)
|
||||
|
||||
|
||||
def test_num_in_flight_tokens_accounting():
|
||||
scheduler = create_scheduler(async_scheduling=True)
|
||||
request = create_requests(num_requests=1, num_tokens=NUM_PROMPT_TOKENS)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
out0 = scheduler.schedule()
|
||||
assert request.num_in_flight_tokens == NUM_PROMPT_TOKENS
|
||||
# Async: decode scheduled before the prefill output is processed.
|
||||
out1 = scheduler.schedule()
|
||||
assert request.num_in_flight_tokens == NUM_PROMPT_TOKENS + 1
|
||||
|
||||
scheduler.update_from_output(out0, _make_model_runner_output(out0))
|
||||
assert request.num_in_flight_tokens == 1
|
||||
scheduler.update_from_output(out1, _make_model_runner_output(out1))
|
||||
assert request.num_in_flight_tokens == 0
|
||||
|
||||
|
||||
def test_swa_free_waits_for_in_flight_step():
|
||||
"""Async: out-of-window blocks stay allocated until the step that still
|
||||
reads them has been processed."""
|
||||
scheduler = _create_swa_scheduler(async_scheduling=True)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
req_id = request.request_id
|
||||
block_pool = scheduler.kv_cache_manager.block_pool
|
||||
|
||||
out0 = scheduler.schedule() # prefill, in flight from here on
|
||||
free_after_prefill = block_pool.get_num_free_blocks()
|
||||
|
||||
# Decode scheduled while the prefill still reads the out-of-window blocks:
|
||||
# they must not be freed yet.
|
||||
out1 = scheduler.schedule()
|
||||
assert _num_null_blocks(scheduler, req_id) == 0
|
||||
assert block_pool.get_num_free_blocks() == free_after_prefill
|
||||
|
||||
# Prefill output processed; the next allocate frees the out-of-window
|
||||
# blocks.
|
||||
scheduler.update_from_output(out0, _make_model_runner_output(out0))
|
||||
scheduler.schedule()
|
||||
assert _num_null_blocks(scheduler, req_id) == NUM_OUT_OF_WINDOW_BLOCKS
|
||||
assert (
|
||||
block_pool.get_num_free_blocks()
|
||||
== free_after_prefill + NUM_OUT_OF_WINDOW_BLOCKS
|
||||
)
|
||||
# Not double-freed on the following steps.
|
||||
scheduler.update_from_output(out1, _make_model_runner_output(out1))
|
||||
scheduler.schedule()
|
||||
assert _num_null_blocks(scheduler, req_id) == NUM_OUT_OF_WINDOW_BLOCKS
|
||||
|
||||
|
||||
def test_swa_free_immediate_when_sync():
|
||||
"""Sync: no in-flight step at schedule time, frees happen at the first
|
||||
decode allocation as before."""
|
||||
scheduler = _create_swa_scheduler(async_scheduling=False)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
req_id = request.request_id
|
||||
|
||||
out0 = scheduler.schedule()
|
||||
scheduler.update_from_output(out0, _make_model_runner_output(out0))
|
||||
assert request.num_in_flight_tokens == 0
|
||||
|
||||
scheduler.schedule()
|
||||
assert _num_null_blocks(scheduler, req_id) == NUM_OUT_OF_WINDOW_BLOCKS
|
||||
|
||||
|
||||
def test_swa_admission_cap_accounts_for_overlapping_batches():
|
||||
spec = SlidingWindowSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=1024,
|
||||
)
|
||||
base = spec.max_admission_blocks_per_request(
|
||||
max_in_flight_tokens=1024, max_model_len=16384
|
||||
)
|
||||
# (1024 - 1 + 1024) tokens -> 128 blocks, +1 for window misalignment.
|
||||
assert base == 129
|
||||
overlapped = spec.max_admission_blocks_per_request(
|
||||
max_in_flight_tokens=2 * 1024, max_model_len=16384
|
||||
)
|
||||
# One extra in-flight chunk is held back: (1024 - 1 + 2 * 1024) tokens.
|
||||
assert overlapped == 193
|
||||
|
||||
|
||||
def test_chunked_local_free_waits_for_in_flight_step():
|
||||
"""Chunked-local attention frees whole chunks left of the current one, and
|
||||
is exposed to the same load-WAR: with async scheduling those chunks must
|
||||
stay allocated until the in-flight step that still reads them settles."""
|
||||
scheduler = _create_chunked_scheduler(async_scheduling=True)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
req_id = request.request_id
|
||||
block_pool = scheduler.kv_cache_manager.block_pool
|
||||
|
||||
out0 = scheduler.schedule() # prefill, in flight from here on
|
||||
free_after_prefill = block_pool.get_num_free_blocks()
|
||||
|
||||
# Decode scheduled while the prefill still reads the out-of-chunk blocks.
|
||||
out1 = scheduler.schedule()
|
||||
assert _num_null_blocks(scheduler, req_id) == 0
|
||||
assert block_pool.get_num_free_blocks() == free_after_prefill
|
||||
|
||||
# Prefill output processed; the next allocate frees the out-of-chunk blocks.
|
||||
scheduler.update_from_output(out0, _make_model_runner_output(out0))
|
||||
scheduler.schedule()
|
||||
assert _num_null_blocks(scheduler, req_id) == NUM_OUT_OF_CHUNK_BLOCKS
|
||||
assert (
|
||||
block_pool.get_num_free_blocks() == free_after_prefill + NUM_OUT_OF_CHUNK_BLOCKS
|
||||
)
|
||||
# Not double-freed on the following steps.
|
||||
scheduler.update_from_output(out1, _make_model_runner_output(out1))
|
||||
scheduler.schedule()
|
||||
assert _num_null_blocks(scheduler, req_id) == NUM_OUT_OF_CHUNK_BLOCKS
|
||||
|
||||
|
||||
def test_chunked_local_admission_cap_accounts_for_overlapping_batches():
|
||||
spec = ChunkedLocalAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
attention_chunk_size=1024,
|
||||
)
|
||||
base = spec.max_admission_blocks_per_request(
|
||||
max_in_flight_tokens=1024, max_model_len=16384
|
||||
)
|
||||
# (1024 + 1024) tokens -> 128 blocks.
|
||||
assert base == 128
|
||||
overlapped = spec.max_admission_blocks_per_request(
|
||||
max_in_flight_tokens=2 * 1024, max_model_len=16384
|
||||
)
|
||||
# One extra in-flight chunk is held back: (1024 + 2 * 1024) tokens.
|
||||
assert overlapped == 192
|
||||
|
||||
|
||||
def test_connector_finish_frees_on_settled_basis():
|
||||
"""The out-of-window prune done at request finish, before the block table
|
||||
is handed to a KV connector (simple CPU offload / NIXL store), must use the
|
||||
same processed-token basis. Otherwise the connector reads/hands off a block
|
||||
the still-in-flight step is optimistically counted as done with, which is
|
||||
the load-path WAR this fix closes."""
|
||||
scheduler = create_scheduler(
|
||||
block_size=BLOCK_SIZE,
|
||||
async_scheduling=True,
|
||||
use_kv_connector=mock_kv(matched_tokens=0, is_async=False),
|
||||
kv_cache_spec=SlidingWindowSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=SLIDING_WINDOW,
|
||||
),
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
req_id = request.request_id
|
||||
|
||||
out0 = scheduler.schedule() # prefill, in flight from here on
|
||||
scheduler.schedule() # decode over-scheduled: num_computed_tokens optimistic
|
||||
|
||||
# Finishing now (connector store) must NOT prune out-of-window blocks the
|
||||
# in-flight prefill still reads.
|
||||
scheduler._connector_finished(request)
|
||||
assert _num_null_blocks(scheduler, req_id) == 0
|
||||
|
||||
# Once the in-flight step settles, the same prune releases them.
|
||||
scheduler.update_from_output(out0, _make_model_runner_output(out0))
|
||||
scheduler._connector_finished(request)
|
||||
assert _num_null_blocks(scheduler, req_id) == NUM_OUT_OF_WINDOW_BLOCKS
|
||||
+10
-11
@@ -30,6 +30,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
@@ -63,6 +64,7 @@ def create_scheduler(
|
||||
use_ec_connector: bool = False,
|
||||
ec_role: str | None = None,
|
||||
use_v2_model_runner: bool | None = None,
|
||||
kv_cache_spec: KVCacheSpec | None = None,
|
||||
) -> Scheduler | AsyncScheduler:
|
||||
"""Create scheduler under test.
|
||||
|
||||
@@ -164,20 +166,17 @@ def create_scheduler(
|
||||
speculative_config=speculative_config,
|
||||
ec_transfer_config=ec_transfer_config,
|
||||
)
|
||||
if kv_cache_spec is None:
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_blocks, # A large number of blocks to hold all requests
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer"], kv_cache_spec)],
|
||||
)
|
||||
cache_config.num_gpu_blocks = num_blocks
|
||||
register_all_kvcache_specs(vllm_config)
|
||||
|
||||
@@ -39,13 +39,18 @@ from vllm.v1.worker.mamba_utils import get_mamba_groups
|
||||
class StepAction:
|
||||
num_computed_tokens_start: int
|
||||
num_scheduled_tokens: int
|
||||
kv_cache_block_ids: list[int] # [] to follow last step
|
||||
kv_cache_block_ids: list[int] # per-block mask: 1=held, 0=freed/nulled
|
||||
preprocess_copy_idx: tuple[int, int] # -1, -1 for no copy
|
||||
postprocess_copy_idx: tuple[int, int] # -1, -1 for no copy
|
||||
|
||||
|
||||
num_speculative_tokens = 3
|
||||
|
||||
# Whether the run under test uses async scheduling. Set by each test entrypoint
|
||||
# before generation; consulted where the scheduler's optimistic token count must
|
||||
# be corrected for in-flight (possibly-rejected) speculative tokens.
|
||||
async_scheduling_mode = False
|
||||
|
||||
num_accepted_tokens = 1
|
||||
prompt_token_ids: list[int] = []
|
||||
MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8"
|
||||
@@ -249,7 +254,8 @@ def get_fake_execute_model_fn(original_execute_model_fn: Callable):
|
||||
scheduler_output.scheduled_cached_reqs.num_computed_tokens[0]
|
||||
)
|
||||
if (
|
||||
self.num_spec_tokens
|
||||
async_scheduling_mode
|
||||
and self.num_spec_tokens
|
||||
and num_prompt_tokens is not None
|
||||
and num_computed_tokens > num_prompt_tokens
|
||||
):
|
||||
@@ -496,7 +502,10 @@ def apply_patch(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(mamba_utils, "do_mamba_copy_block", fake_copy_fn)
|
||||
|
||||
|
||||
def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
def get_mamba_prefix_cache_step_configs(
|
||||
async_scheduling: bool = False,
|
||||
) -> dict[str, TestConfig]:
|
||||
a = async_scheduling
|
||||
tests = {
|
||||
"accept_1": TestConfig(
|
||||
num_prompt_tokens=554,
|
||||
@@ -504,14 +513,24 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=1,
|
||||
step_actions=[
|
||||
StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(556, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(557, 4, [], (0, 1), (-1, -1)),
|
||||
StepAction(558, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(559, 4, [], (-1, -1), (1, 0)),
|
||||
StepAction(560, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(561, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
555, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1)
|
||||
),
|
||||
StepAction(
|
||||
556, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1)
|
||||
),
|
||||
StepAction(557, 4, [1, 1, 1, 1, 1], (0, 1), (-1, -1)),
|
||||
StepAction(558, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(559, 4, [1, 1, 1, 1, 1], (-1, -1), (1, 0)),
|
||||
StepAction(560, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
561,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
# test case 2.1: no hit, accept 2 tokens
|
||||
@@ -521,11 +540,19 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=2,
|
||||
step_actions=[
|
||||
StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(556, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(558, 4, [], (1, 1), (2, 0)),
|
||||
StepAction(560, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(562, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
556, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1)
|
||||
),
|
||||
StepAction(558, 4, [1, 1, 1, 1, 1], (1, 1), (2, 0)),
|
||||
StepAction(560, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
562,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
# test case 2.2: no hit, accept 2 tokens
|
||||
@@ -535,10 +562,16 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=2,
|
||||
step_actions=[
|
||||
StepAction(0, 555, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(557, 4, [1, 1, 1, 1, 1], (1, 1), (-1, -1)),
|
||||
StepAction(559, 4, [], (-1, -1), (1, 0)),
|
||||
StepAction(561, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(559, 4, [1, 1, 1, 1, 1], (-1, -1), (1, 0)),
|
||||
StepAction(
|
||||
561,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(563, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
@@ -548,10 +581,18 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=3,
|
||||
step_actions=[
|
||||
StepAction(0, 553, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(553, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(556, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(559, 4, [], (2, 1), (1, 0)),
|
||||
StepAction(562, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(553, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
556, 4, [1, 1, 1, 1, 1] if a else [1, 1, 1, 1], (-1, -1), (-1, -1)
|
||||
),
|
||||
StepAction(559, 4, [1, 1, 1, 1, 1], (2, 1), (1, 0)),
|
||||
StepAction(
|
||||
562,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(565, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
@@ -561,10 +602,16 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=3,
|
||||
step_actions=[
|
||||
StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(557, 4, [1, 1, 1, 1, 1], (2, 1), (3, 0)),
|
||||
StepAction(560, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(563, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(560, 4, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
563,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"accept_3_3": TestConfig(
|
||||
@@ -573,9 +620,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=3,
|
||||
step_actions=[
|
||||
StepAction(0, 555, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(558, 4, [1, 1, 1, 1, 1], (2, 1), (2, 0)),
|
||||
StepAction(561, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
561,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(564, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
@@ -585,9 +638,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, 553, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(553, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(553, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(557, 4, [1, 1, 1, 1, 1], (3, 1), (3, 0)),
|
||||
StepAction(561, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
561,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(565, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
@@ -597,9 +656,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, 554, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(554, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(558, 4, [1, 1, 1, 1, 1], (3, 1), (2, 0)),
|
||||
StepAction(562, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
562,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(566, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
@@ -609,9 +674,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, 555, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(555, 4, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(559, 4, [1, 1, 1, 1, 1], (3, 1), (1, 0)),
|
||||
StepAction(563, 4, [], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
563,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(567, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
@@ -621,9 +692,15 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, 556, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(556, 4, [], (-1, -1), (3, 0)),
|
||||
StepAction(556, 4, [1, 1, 1, 1], (-1, -1), (3, 0)),
|
||||
StepAction(560, 4, [1, 1, 1, 1, 1], (0, 1), (-1, -1)),
|
||||
StepAction(564, 4, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
564,
|
||||
4,
|
||||
[1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"prompt_block_size": TestConfig(
|
||||
@@ -642,7 +719,13 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
step_actions=[
|
||||
StepAction(0, 560, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(560, 560, [1, 1, 1, 1, 1], (0, 1), (-1, -1)),
|
||||
StepAction(560 * 2, 4, [0, 1, 1, 1, 1, 1], (1, 2), (-1, -1)),
|
||||
StepAction(
|
||||
560 * 2,
|
||||
4,
|
||||
[1, 1, 1, 1, 1, 1] if a else [0, 1, 1, 1, 1, 1],
|
||||
(1, 2),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"prompt_2_block_size_10": TestConfig(
|
||||
@@ -652,7 +735,13 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
step_actions=[
|
||||
StepAction(0, 560, [1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(560, 570, [1, 0, 1, 1, 1, 1], (0, 2), (-1, -1)),
|
||||
StepAction(560 * 2 + 10, 4, [0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
560 * 2 + 10,
|
||||
4,
|
||||
[1, 0, 1, 1, 1, 1] if a else [0, 0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"prompt_3_block_size": TestConfig(
|
||||
@@ -662,7 +751,13 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
step_actions=[
|
||||
StepAction(0, 560 * 2, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(560 * 2, 560, [0, 1, 1, 1, 1, 1], (1, 2), (-1, -1)),
|
||||
StepAction(560 * 3, 4, [0, 0, 1, 1, 1, 1, 1], (2, 3), (-1, -1)),
|
||||
StepAction(
|
||||
560 * 3,
|
||||
4,
|
||||
[0, 1, 1, 1, 1, 1, 1] if a else [0, 0, 1, 1, 1, 1, 1],
|
||||
(2, 3),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"prompt_3_block_size_10": TestConfig(
|
||||
@@ -672,7 +767,13 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
step_actions=[
|
||||
StepAction(0, 560 * 2, [0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(560 * 2, 570, [0, 1, 0, 1, 1, 1, 1], (1, 3), (-1, -1)),
|
||||
StepAction(560 * 3 + 10, 4, [0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
560 * 3 + 10,
|
||||
4,
|
||||
[0, 1, 0, 1, 1, 1, 1] if a else [0, 0, 0, 1, 1, 1, 1],
|
||||
(-1, -1),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"prompt_10_block_size": TestConfig(
|
||||
@@ -691,14 +792,18 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
StepAction(
|
||||
560 * 9,
|
||||
560,
|
||||
[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1]
|
||||
if a
|
||||
else [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
(8, 9),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(
|
||||
560 * 10,
|
||||
4,
|
||||
[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1]
|
||||
if a
|
||||
else [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
(9, 10),
|
||||
(-1, -1),
|
||||
),
|
||||
@@ -720,34 +825,28 @@ def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
StepAction(
|
||||
560 * 9,
|
||||
560 + 10,
|
||||
[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1],
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1]
|
||||
if a
|
||||
else [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1],
|
||||
(8, 10),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
}
|
||||
|
||||
return tests
|
||||
|
||||
|
||||
def fill_following_kv_cache_block_ids(test_config: TestConfig) -> None:
|
||||
for step_action_prev, step_action_next in zip(
|
||||
test_config.step_actions[:-1], test_config.step_actions[1:]
|
||||
):
|
||||
if len(step_action_next.kv_cache_block_ids) == 0:
|
||||
step_action_next.kv_cache_block_ids = (
|
||||
step_action_prev.kv_cache_block_ids.copy()
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
def _run_mamba_prefix_cache_mrv1(
|
||||
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
|
||||
):
|
||||
global async_scheduling_mode
|
||||
async_scheduling_mode = async_scheduling
|
||||
run_ref_mamba_state_in_subprocess()
|
||||
apply_patch(monkeypatch)
|
||||
prompt_dataset = datasets.load_dataset("heheda/a_long_article")
|
||||
full_prompt = prompt_dataset["train"][0]["text"]
|
||||
tests = get_mamba_prefix_cache_step_configs()
|
||||
tests = get_mamba_prefix_cache_step_configs(async_scheduling)
|
||||
|
||||
engine = LLM(
|
||||
model=MODEL,
|
||||
@@ -761,6 +860,7 @@ def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
},
|
||||
max_num_batched_tokens=3072,
|
||||
hf_overrides={"num_hidden_layers": NUM_HIDDEN_LAYERS},
|
||||
async_scheduling=async_scheduling,
|
||||
seed=42,
|
||||
)
|
||||
global prompt_token_ids
|
||||
@@ -777,7 +877,6 @@ def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
)
|
||||
global cur_step_action_idx
|
||||
cur_step_action_idx = 0
|
||||
fill_following_kv_cache_block_ids(test_config)
|
||||
global step_actions
|
||||
step_actions = test_config.step_actions
|
||||
_ = engine.generate(
|
||||
@@ -800,7 +899,20 @@ def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=False)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv1(monkeypatch, async_scheduling=True)
|
||||
|
||||
|
||||
def _run_mamba_prefix_cache_mrv2(
|
||||
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
|
||||
):
|
||||
global async_scheduling_mode
|
||||
async_scheduling_mode = async_scheduling
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
envs.disable_envs_cache()
|
||||
@@ -998,11 +1110,12 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
max_num_batched_tokens=3072,
|
||||
max_model_len=BLOCK_SIZE * 12,
|
||||
hf_overrides={"num_hidden_layers": NUM_HIDDEN_LAYERS},
|
||||
async_scheduling=async_scheduling,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
try:
|
||||
tests = get_mamba_prefix_cache_step_configs()
|
||||
tests = get_mamba_prefix_cache_step_configs(async_scheduling)
|
||||
|
||||
global step_actions
|
||||
global cur_step_action_idx
|
||||
@@ -1010,7 +1123,6 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
for test_name, test_config in tests.items():
|
||||
num_accepted_tokens = test_config.num_accepted_tokens
|
||||
cur_step_action_idx = 0
|
||||
fill_following_kv_cache_block_ids(test_config)
|
||||
step_actions = test_config.step_actions
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
@@ -1053,3 +1165,13 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
del engine
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv2(monkeypatch, async_scheduling=False)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv2_async(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv2(monkeypatch, async_scheduling=True)
|
||||
|
||||
Reference in New Issue
Block a user