[BugFix] Fix VllmConfig() construction on all platforms (#20695)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -116,11 +116,13 @@ class TpuPlatform(Platform):
|
||||
assert vllm_config.speculative_config is None, \
|
||||
"TPU does not support speculative decoding"
|
||||
|
||||
if vllm_config.model_config.dtype in (torch.float16, torch.float32):
|
||||
model_config = vllm_config.model_config
|
||||
if model_config is not None and model_config.dtype in (torch.float16,
|
||||
torch.float32):
|
||||
logger.warning(
|
||||
"The TPU backend currently does not support %s. "
|
||||
"Using bfloat16 instead.", vllm_config.model_config.dtype)
|
||||
vllm_config.model_config.dtype = torch.bfloat16
|
||||
"Using bfloat16 instead.", model_config.dtype)
|
||||
model_config.dtype = torch.bfloat16
|
||||
|
||||
from vllm.v1.attention.backends.pallas import PallasAttentionBackend
|
||||
cache_config.block_size = PallasAttentionBackend.get_page_size(
|
||||
@@ -146,7 +148,7 @@ class TpuPlatform(Platform):
|
||||
"Forcing --disable_chunked_mm_input.")
|
||||
scheduler_config.disable_chunked_mm_input = True
|
||||
|
||||
if vllm_config.model_config and vllm_config.model_config.use_mla:
|
||||
if model_config and model_config.use_mla:
|
||||
logger.info(
|
||||
"MLA is enabled on a non-GPU platform; forcing chunked "
|
||||
"prefill and prefix caching to be disabled.")
|
||||
|
||||
Reference in New Issue
Block a user