diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py
index 5146a36225..4fcb3ca17d 100644
--- a/vllm/config/vllm.py
+++ b/vllm/config/vllm.py
@@ -1147,6 +1147,19 @@ class VllmConfig:
                     "connectors (PD disaggregation, KV cache offload)."
                 )
 
+        if (
+            self.model_config is not None
+            and self.model_config.multimodal_config is not None
+            and self.model_config.multimodal_config.language_model_only
+            and self.compilation_config.cudagraph_mm_encoder
+        ):
+            raise ValueError(
+                "--language-model-only is incompatible with "
+                "cudagraph_mm_encoder=True, since it disables all multimodal "
+                "inputs and the multimodal encoder is never run. Please "
+                "disable one of them."
+            )
+
         self._verify_sampling_replay_config()
         self._verify_trace_replay_config()