[Bugfix][V1] Free out-of-window blocks on the processed-token basis under async scheduling (#47728)

Signed-off-by: Saddss <28726669061@qq.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Saddss <28726669061@qq.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Saddss
2026-07-08 13:34:19 +01:00
committed by GitHub
co-authored by Yifan Qiao Saddss Nick Hill
parent 934eeaecfb
commit cd0de48d08
14 changed files with 549 additions and 143 deletions
+18 -15
View File
@@ -482,25 +482,28 @@ class ChunkedLocalAttentionSpec(AttentionSpec):
attention_chunk_size: int
def max_admission_blocks_per_request(
self, max_num_batched_tokens: int, max_model_len: int
self, max_in_flight_tokens: int, max_model_len: int
) -> int:
"""Per-request admission cap, in blocks.
Single source of truth for both startup pool sizing
(`max_memory_usage_bytes`) and the runtime admission gate, so requests
admitted by startup can also be admitted at runtime.
`max_in_flight_tokens` is the max tokens scheduled but not yet settled
(one batch per concurrent step); see `VllmConfig.max_in_flight_tokens`.
"""
# During chunked prefill, we hold KV for at most one chunk window.
# During chunked prefill, we hold KV for at most one chunk window plus
# the in-flight tokens, since frees happen on the processed-token basis.
num_tokens = min(
self.attention_chunk_size + max_num_batched_tokens, max_model_len
self.attention_chunk_size + max_in_flight_tokens, max_model_len
)
return cdiv(num_tokens, self.block_size)
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
max_model_len = vllm_config.model_config.max_model_len
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
max_blocks = self.max_admission_blocks_per_request(
max_num_batched_tokens=max_num_batched_tokens, max_model_len=max_model_len
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
max_model_len=vllm_config.model_config.max_model_len,
)
return max_blocks * self.page_size_bytes
@@ -544,7 +547,7 @@ class SlidingWindowSpec(AttentionSpec):
)
def max_admission_blocks_per_request(
self, max_num_batched_tokens: int, max_model_len: int
self, max_in_flight_tokens: int, max_model_len: int
) -> int:
"""Per-request admission cap, in blocks.
@@ -553,13 +556,14 @@ class SlidingWindowSpec(AttentionSpec):
real-held blocks plateau at this bound because
`SlidingWindowManager.remove_skipped_blocks` runs from `allocate_slots`
before each chunk's `get_num_blocks_to_allocate`.
`max_in_flight_tokens` is the max tokens scheduled but not yet settled
(one batch per concurrent step); see `VllmConfig.max_in_flight_tokens`.
"""
# During chunked prefill, we hold KV for the last `sliding_window-1`
# computed tokens plus the newly scheduled tokens, and never more
# than `max_model_len`.
num_tokens = min(
self.sliding_window - 1 + max_num_batched_tokens, max_model_len
)
# computed tokens plus the in-flight tokens (frees happen on the
# processed-token basis); never more than `max_model_len`.
num_tokens = min(self.sliding_window - 1 + max_in_flight_tokens, max_model_len)
# +1 because the sliding window may not start from the beginning of
# the block. E.g. block size 4 and num_token 4 needs two blocks
# [XXCD][EF] to store the 6-token window [CDEF].
@@ -569,10 +573,9 @@ class SlidingWindowSpec(AttentionSpec):
assert vllm_config.parallel_config.decode_context_parallel_size == 1, (
"DCP not support sliding window."
)
max_model_len = vllm_config.model_config.max_model_len
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
max_blocks = self.max_admission_blocks_per_request(
max_num_batched_tokens=max_num_batched_tokens, max_model_len=max_model_len
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
max_model_len=vllm_config.model_config.max_model_len,
)
return max_blocks * self.page_size_bytes