From b4ac4fa04da14c4f40688fb98211367981a1f4d7 Mon Sep 17 00:00:00 2001 From: Lucia Fang <116399278+luccafong@users.noreply.github.com> Date: Mon, 28 Apr 2025 19:22:22 -0700 Subject: [PATCH] [model] make llama4 compatible with pure dense layers (#17315) Signed-off-by: Lucia Fang --- vllm/model_executor/models/llama4.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/models/llama4.py b/vllm/model_executor/models/llama4.py index e5d1a671f5..0fdc30f36f 100644 --- a/vllm/model_executor/models/llama4.py +++ b/vllm/model_executor/models/llama4.py @@ -273,8 +273,8 @@ class Llama4DecoderLayer(nn.Module): cache_config=cache_config, prefix=f"{prefix}.self_attn", ) - is_moe_layer = (self.layer_idx + - 1) % config.interleave_moe_layer_step == 0 + is_moe_layer = config.interleave_moe_layer_step > 0 and ( + self.layer_idx + 1) % config.interleave_moe_layer_step == 0 if is_moe_layer: self.feed_forward = Llama4MoE( config=config,