[Bugfix][KV Offload] Handle chunked local attention in offloading scheduler (#51161)
Signed-off-by: almogtavor <almogtavor@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Or Ozeri
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
1b0ce31f32
commit
eb24bc38cf
@@ -31,6 +31,7 @@ from vllm.utils.math_utils import cdiv, round_down
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheSpec,
|
||||
@@ -112,6 +113,11 @@ def get_sliding_window_size_in_chunks(
|
||||
assert kv_cache_spec.sliding_window > 0
|
||||
return cdiv(kv_cache_spec.sliding_window, tokens_per_chunk)
|
||||
|
||||
if isinstance(kv_cache_spec, ChunkedLocalAttentionSpec):
|
||||
# Attention never reaches back past one chunk
|
||||
assert kv_cache_spec.attention_chunk_size > 0
|
||||
return cdiv(kv_cache_spec.attention_chunk_size, tokens_per_chunk)
|
||||
|
||||
if isinstance(kv_cache_spec, MambaSpec):
|
||||
# Mamba depends on a single state
|
||||
return 1
|
||||
|
||||
Reference in New Issue
Block a user