[Model] Drop the 4x M-RoPE cache headroom for Qwen3-VL, Qwen3.5 and Qwen3.8 (#59853)
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com> Signed-off-by: lesj0610 <lesj0610@gmail.com> Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
@@ -79,6 +79,7 @@ class Qwen3Attention(nn.Module):
|
||||
attn_type: str = AttentionType.DECODER,
|
||||
dual_chunk_attention_config: dict[str, Any] | None = None,
|
||||
per_layer_sliding_window: int | None = None,
|
||||
mrope_positions_factor: int = 4,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -124,6 +125,7 @@ class Qwen3Attention(nn.Module):
|
||||
max_position=max_position,
|
||||
rope_parameters=rope_parameters,
|
||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||
mrope_positions_factor=mrope_positions_factor,
|
||||
)
|
||||
attn_cls = (
|
||||
EncoderOnlyAttention
|
||||
@@ -179,6 +181,7 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
per_layer_sliding_window: int | None = None,
|
||||
mrope_positions_factor: int = 4,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
@@ -211,6 +214,7 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
attn_type=attn_type,
|
||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||
per_layer_sliding_window=per_layer_sliding_window,
|
||||
mrope_positions_factor=mrope_positions_factor,
|
||||
)
|
||||
self.mlp = Qwen3MLP(
|
||||
hidden_size=self.hidden_size,
|
||||
@@ -263,9 +267,13 @@ ALL_DECODER_LAYER_TYPES = {
|
||||
}
|
||||
)
|
||||
class Qwen3Model(Qwen2Model):
|
||||
decoder_layer_type: type[nn.Module] = Qwen3DecoderLayer
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(
|
||||
vllm_config=vllm_config, prefix=prefix, decoder_layer_type=Qwen3DecoderLayer
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
decoder_layer_type=self.decoder_layer_type,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user