[Bugfix][V1] Free out-of-window blocks on the processed-token basis under async scheduling (#47728)
Signed-off-by: Saddss <28726669061@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Saddss <28726669061@qq.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Yifan Qiao
Saddss
Nick Hill
parent
934eeaecfb
commit
cd0de48d08
@@ -482,25 +482,28 @@ class ChunkedLocalAttentionSpec(AttentionSpec):
|
||||
attention_chunk_size: int
|
||||
|
||||
def max_admission_blocks_per_request(
|
||||
self, max_num_batched_tokens: int, max_model_len: int
|
||||
self, max_in_flight_tokens: int, max_model_len: int
|
||||
) -> int:
|
||||
"""Per-request admission cap, in blocks.
|
||||
|
||||
Single source of truth for both startup pool sizing
|
||||
(`max_memory_usage_bytes`) and the runtime admission gate, so requests
|
||||
admitted by startup can also be admitted at runtime.
|
||||
|
||||
`max_in_flight_tokens` is the max tokens scheduled but not yet settled
|
||||
(one batch per concurrent step); see `VllmConfig.max_in_flight_tokens`.
|
||||
"""
|
||||
# During chunked prefill, we hold KV for at most one chunk window.
|
||||
# During chunked prefill, we hold KV for at most one chunk window plus
|
||||
# the in-flight tokens, since frees happen on the processed-token basis.
|
||||
num_tokens = min(
|
||||
self.attention_chunk_size + max_num_batched_tokens, max_model_len
|
||||
self.attention_chunk_size + max_in_flight_tokens, max_model_len
|
||||
)
|
||||
return cdiv(num_tokens, self.block_size)
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
max_blocks = self.max_admission_blocks_per_request(
|
||||
max_num_batched_tokens=max_num_batched_tokens, max_model_len=max_model_len
|
||||
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
|
||||
max_model_len=vllm_config.model_config.max_model_len,
|
||||
)
|
||||
return max_blocks * self.page_size_bytes
|
||||
|
||||
@@ -544,7 +547,7 @@ class SlidingWindowSpec(AttentionSpec):
|
||||
)
|
||||
|
||||
def max_admission_blocks_per_request(
|
||||
self, max_num_batched_tokens: int, max_model_len: int
|
||||
self, max_in_flight_tokens: int, max_model_len: int
|
||||
) -> int:
|
||||
"""Per-request admission cap, in blocks.
|
||||
|
||||
@@ -553,13 +556,14 @@ class SlidingWindowSpec(AttentionSpec):
|
||||
real-held blocks plateau at this bound because
|
||||
`SlidingWindowManager.remove_skipped_blocks` runs from `allocate_slots`
|
||||
before each chunk's `get_num_blocks_to_allocate`.
|
||||
|
||||
`max_in_flight_tokens` is the max tokens scheduled but not yet settled
|
||||
(one batch per concurrent step); see `VllmConfig.max_in_flight_tokens`.
|
||||
"""
|
||||
# During chunked prefill, we hold KV for the last `sliding_window-1`
|
||||
# computed tokens plus the newly scheduled tokens, and never more
|
||||
# than `max_model_len`.
|
||||
num_tokens = min(
|
||||
self.sliding_window - 1 + max_num_batched_tokens, max_model_len
|
||||
)
|
||||
# computed tokens plus the in-flight tokens (frees happen on the
|
||||
# processed-token basis); never more than `max_model_len`.
|
||||
num_tokens = min(self.sliding_window - 1 + max_in_flight_tokens, max_model_len)
|
||||
# +1 because the sliding window may not start from the beginning of
|
||||
# the block. E.g. block size 4 and num_token 4 needs two blocks
|
||||
# [XXCD][EF] to store the 6-token window [CDEF].
|
||||
@@ -569,10 +573,9 @@ class SlidingWindowSpec(AttentionSpec):
|
||||
assert vllm_config.parallel_config.decode_context_parallel_size == 1, (
|
||||
"DCP not support sliding window."
|
||||
)
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
max_blocks = self.max_admission_blocks_per_request(
|
||||
max_num_batched_tokens=max_num_batched_tokens, max_model_len=max_model_len
|
||||
max_in_flight_tokens=vllm_config.max_in_flight_tokens,
|
||||
max_model_len=vllm_config.model_config.max_model_len,
|
||||
)
|
||||
return max_blocks * self.page_size_bytes
|
||||
|
||||
|
||||
Reference in New Issue
Block a user