[Core] Support Local Chunked Attention for Hybrid KV Cache (#19351)
Signed-off-by: Lucia Fang <fanglu@fb.com> Signed-off-by: Lu Fang <fanglu@meta.com> Signed-off-by: Lu Fang <fanglu@fb.com> Co-authored-by: Lu Fang <fanglu@meta.com>
This commit is contained in:
@@ -862,6 +862,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
use_sliding_window = (isinstance(kv_cache_spec, SlidingWindowSpec) or
|
||||
(isinstance(kv_cache_spec, FullAttentionSpec)
|
||||
and kv_cache_spec.sliding_window is not None))
|
||||
use_local_attention = (
|
||||
isinstance(kv_cache_spec, ChunkedLocalAttentionSpec)
|
||||
or (isinstance(kv_cache_spec, FullAttentionSpec)
|
||||
and kv_cache_spec.attention_chunk_size is not None))
|
||||
assert isinstance(kv_cache_spec, AttentionSpec)
|
||||
use_cascade = attn_metadata_builder.use_cascade_attention(
|
||||
common_prefix_len=common_prefix_len,
|
||||
@@ -870,6 +874,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_kv_heads=kv_cache_spec.num_kv_heads,
|
||||
use_alibi=self.use_alibi,
|
||||
use_sliding_window=use_sliding_window,
|
||||
use_local_attention=use_local_attention,
|
||||
num_sms=self.num_sms,
|
||||
)
|
||||
return common_prefix_len if use_cascade else 0
|
||||
@@ -2672,6 +2677,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
dtype=self.kv_cache_dtype,
|
||||
sliding_window=attn_module.sliding_window,
|
||||
use_mla=use_mla)
|
||||
assert not use_local_attention, (
|
||||
"attention module can not be with ",
|
||||
"both local attention and sliding window")
|
||||
elif use_local_attention:
|
||||
kv_cache_spec[layer_name] = (ChunkedLocalAttentionSpec(
|
||||
block_size=block_size,
|
||||
|
||||
Reference in New Issue
Block a user