[Model] Drop the 4x M-RoPE cache headroom for Qwen3-VL, Qwen3.5 and Qwen3.8 (#59853)

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
Signed-off-by: lesj0610 <lesj0610@gmail.com>
Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
SeongJun Lee
2026-10-10 06:51:52 +00:00
committed by GitHub
co-authored by lesj0610
parent c1692c85f1
commit a98247ab4d
11 changed files with 56 additions and 6 deletions
+9 -1
View File
@@ -79,6 +79,7 @@ class Qwen3Attention(nn.Module):
attn_type: str = AttentionType.DECODER,
dual_chunk_attention_config: dict[str, Any] | None = None,
per_layer_sliding_window: int | None = None,
mrope_positions_factor: int = 4,
) -> None:
super().__init__()
self.hidden_size = hidden_size
@@ -124,6 +125,7 @@ class Qwen3Attention(nn.Module):
max_position=max_position,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
mrope_positions_factor=mrope_positions_factor,
)
attn_cls = (
EncoderOnlyAttention
@@ -179,6 +181,7 @@ class Qwen3DecoderLayer(nn.Module):
quant_config: QuantizationConfig | None = None,
prefix: str = "",
per_layer_sliding_window: int | None = None,
mrope_positions_factor: int = 4,
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
@@ -211,6 +214,7 @@ class Qwen3DecoderLayer(nn.Module):
attn_type=attn_type,
dual_chunk_attention_config=dual_chunk_attention_config,
per_layer_sliding_window=per_layer_sliding_window,
mrope_positions_factor=mrope_positions_factor,
)
self.mlp = Qwen3MLP(
hidden_size=self.hidden_size,
@@ -263,9 +267,13 @@ ALL_DECODER_LAYER_TYPES = {
}
)
class Qwen3Model(Qwen2Model):
decoder_layer_type: type[nn.Module] = Qwen3DecoderLayer
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(
vllm_config=vllm_config, prefix=prefix, decoder_layer_type=Qwen3DecoderLayer
vllm_config=vllm_config,
prefix=prefix,
decoder_layer_type=self.decoder_layer_type,
)