[Frontend][TPU] Add TPU default max-num-batched-tokens based on device name (#17508)

Signed-off-by: Chenyaaang <chenyangli@google.com>
This commit is contained in:
Chenyaaang
2025-05-02 21:42:44 -07:00
committed by GitHub
parent e3d0a1d190
commit 87baebebd8
3 changed files with 40 additions and 4 deletions
+3 -1
View File
@@ -3,6 +3,7 @@
from typing import TYPE_CHECKING, Optional, Union
import torch
from tpu_info import device
import vllm.envs as envs
from vllm.inputs import ProcessorInputs, PromptType
@@ -54,7 +55,8 @@ class TpuPlatform(Platform):
@classmethod
def get_device_name(cls, device_id: int = 0) -> str:
return "tpu"
chip_type, _ = device.get_local_chips()
return f"TPU {chip_type.name}"
@classmethod
def get_device_total_memory(cls, device_id: int = 0) -> int: