diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py
index 2b2ff4bab8..4cc433cbfe 100644
--- a/vllm/v1/engine/input_processor.py
+++ b/vllm/v1/engine/input_processor.py
@@ -438,7 +438,7 @@ class InputProcessor:
         prompt_len: int,
         prompt_type: Literal["encoder", "decoder"],
     ):
-        if self.skip_prompt_length_check:
+        if self.skip_prompt_length_check and prompt_type == "encoder":
             return
 
         if prompt_len == 0 and prompt_type == "decoder":