diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 88e6660e21..52c04509b2 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1617,7 +1617,7 @@ class VllmConfig: max_size = rocm_aiter_ops.get_aiter_allreduce_max_size() else: max_size = compilation_config.pass_config.flashinfer_max_size(tp_size) - if max_size is not None: + if max_size is not None and self.model_config is not None: assert isinstance(self.model_config.dtype, torch.dtype) max_token_num = max_size // ( self.model_config.get_hidden_size()