[Core] Support Local Chunked Attention for Hybrid KV Cache (#19351)

Signed-off-by: Lucia Fang <fanglu@fb.com>
Signed-off-by: Lu Fang <fanglu@meta.com>
Signed-off-by: Lu Fang <fanglu@fb.com>
Co-authored-by: Lu Fang <fanglu@meta.com>
This commit is contained in:
Lucia Fang
2025-07-18 20:48:38 -07:00
committed by GitHub
co-authored by Lu Fang
parent 466e878f2a
commit 9a9fda1423
9 changed files with 351 additions and 19 deletions
+8
View File
@@ -862,6 +862,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
use_sliding_window = (isinstance(kv_cache_spec, SlidingWindowSpec) or
(isinstance(kv_cache_spec, FullAttentionSpec)
and kv_cache_spec.sliding_window is not None))
use_local_attention = (
isinstance(kv_cache_spec, ChunkedLocalAttentionSpec)
or (isinstance(kv_cache_spec, FullAttentionSpec)
and kv_cache_spec.attention_chunk_size is not None))
assert isinstance(kv_cache_spec, AttentionSpec)
use_cascade = attn_metadata_builder.use_cascade_attention(
common_prefix_len=common_prefix_len,
@@ -870,6 +874,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_kv_heads=kv_cache_spec.num_kv_heads,
use_alibi=self.use_alibi,
use_sliding_window=use_sliding_window,
use_local_attention=use_local_attention,
num_sms=self.num_sms,
)
return common_prefix_len if use_cascade else 0
@@ -2672,6 +2677,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
dtype=self.kv_cache_dtype,
sliding_window=attn_module.sliding_window,
use_mla=use_mla)
assert not use_local_attention, (
"attention module can not be with ",
"both local attention and sliding window")
elif use_local_attention:
kv_cache_spec[layer_name] = (ChunkedLocalAttentionSpec(
block_size=block_size,