[Attention] Clean up iRoPE in V1 (#21188)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Michael Goin
parent
6ece16c4fe
commit
304dce7ec0
@@ -2702,8 +2702,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# TODO: Support other attention modules, e.g., cross-attention
|
||||
if attn_module.attn_type == AttentionType.DECODER:
|
||||
use_local_attention = (self.attention_chunk_size is not None
|
||||
and getattr(attn_module.impl,
|
||||
"use_irope", False))
|
||||
and attn_module.use_irope)
|
||||
if attn_module.sliding_window is not None:
|
||||
kv_cache_spec[layer_name] = SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
@@ -2716,13 +2715,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
"attention module can not be with ",
|
||||
"both local attention and sliding window")
|
||||
elif use_local_attention:
|
||||
kv_cache_spec[layer_name] = (ChunkedLocalAttentionSpec(
|
||||
kv_cache_spec[layer_name] = ChunkedLocalAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=attn_module.num_kv_heads,
|
||||
head_size=attn_module.head_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
attention_chunk_size=self.attention_chunk_size,
|
||||
use_mla=use_mla))
|
||||
use_mla=use_mla)
|
||||
else:
|
||||
kv_cache_spec[layer_name] = FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
|
||||
Reference in New Issue
Block a user