diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py
index 2e0126368c..3dc8d33cf9 100644
--- a/vllm/config/speculative.py
+++ b/vllm/config/speculative.py
@@ -306,8 +306,10 @@ class SpeculativeConfig:
         )
         factors.append(uses_aux_hidden_states)
 
-        # The specific layers used also affect the computation graph
         if uses_aux_hidden_states and self.draft_model_config is not None:
+            factors.append(self.draft_model_config.compute_hash())
+
+            # The specific layers used also affect the computation graph.
             layer_ids = getattr(
                 self.draft_model_config.hf_config,
                 "eagle_aux_hidden_state_layer_ids",