Added sliding window attention support for qwen-eagle3 architecture (#47568)
Signed-off-by: shanjiaz <zsjwpianpian@gmail.com>
This commit is contained in:
@@ -78,6 +78,7 @@ class Qwen3Attention(nn.Module):
|
|||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
attn_type: str = AttentionType.DECODER,
|
attn_type: str = AttentionType.DECODER,
|
||||||
dual_chunk_attention_config: dict[str, Any] | None = None,
|
dual_chunk_attention_config: dict[str, Any] | None = None,
|
||||||
|
per_layer_sliding_window: int | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.hidden_size = hidden_size
|
self.hidden_size = hidden_size
|
||||||
@@ -136,6 +137,7 @@ class Qwen3Attention(nn.Module):
|
|||||||
num_kv_heads=self.num_kv_heads,
|
num_kv_heads=self.num_kv_heads,
|
||||||
cache_config=cache_config,
|
cache_config=cache_config,
|
||||||
quant_config=quant_config,
|
quant_config=quant_config,
|
||||||
|
per_layer_sliding_window=per_layer_sliding_window,
|
||||||
prefix=f"{prefix}.attn",
|
prefix=f"{prefix}.attn",
|
||||||
attn_type=attn_type,
|
attn_type=attn_type,
|
||||||
**{
|
**{
|
||||||
@@ -175,6 +177,7 @@ class Qwen3DecoderLayer(nn.Module):
|
|||||||
cache_config: CacheConfig | None = None,
|
cache_config: CacheConfig | None = None,
|
||||||
quant_config: QuantizationConfig | None = None,
|
quant_config: QuantizationConfig | None = None,
|
||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
|
per_layer_sliding_window: int | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.hidden_size = config.hidden_size
|
self.hidden_size = config.hidden_size
|
||||||
@@ -206,6 +209,7 @@ class Qwen3DecoderLayer(nn.Module):
|
|||||||
prefix=f"{prefix}.self_attn",
|
prefix=f"{prefix}.self_attn",
|
||||||
attn_type=attn_type,
|
attn_type=attn_type,
|
||||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||||
|
per_layer_sliding_window=per_layer_sliding_window,
|
||||||
)
|
)
|
||||||
self.mlp = Qwen3MLP(
|
self.mlp = Qwen3MLP(
|
||||||
hidden_size=self.hidden_size,
|
hidden_size=self.hidden_size,
|
||||||
|
|||||||
@@ -43,11 +43,22 @@ class Qwen3Eagle3DecoderLayer(Qwen3DecoderLayer):
|
|||||||
cache_config = vllm_config.cache_config
|
cache_config = vllm_config.cache_config
|
||||||
quant_config = get_draft_quant_config(vllm_config)
|
quant_config = get_draft_quant_config(vllm_config)
|
||||||
|
|
||||||
|
# Resolve per-layer sliding window from draft config
|
||||||
|
sliding_window = None
|
||||||
|
layer_types = getattr(config, "layer_types", None)
|
||||||
|
if (
|
||||||
|
layer_types
|
||||||
|
and layer_idx < len(layer_types)
|
||||||
|
and layer_types[layer_idx] == "sliding_attention"
|
||||||
|
):
|
||||||
|
sliding_window = getattr(config, "sliding_window", None)
|
||||||
|
|
||||||
super().__init__(
|
super().__init__(
|
||||||
config=config,
|
config=config,
|
||||||
cache_config=cache_config,
|
cache_config=cache_config,
|
||||||
quant_config=quant_config,
|
quant_config=quant_config,
|
||||||
prefix=prefix,
|
prefix=prefix,
|
||||||
|
per_layer_sliding_window=sliding_window,
|
||||||
)
|
)
|
||||||
|
|
||||||
# First layer uses 2*hidden_size (embeds + hidden_states concatenated)
|
# First layer uses 2*hidden_size (embeds + hidden_states concatenated)
|
||||||
|
|||||||
Reference in New Issue
Block a user