[Platform] Add custom default max tokens (#18557)

Signed-off-by: Gabriel Marinho <gmarinho@ibm.com>
This commit is contained in:
Gabriel Marinho
2025-07-04 10:50:17 +08:00
committed by GitHub
parent 7e1665b089
commit a4113b035c
5 changed files with 59 additions and 60 deletions
+4
View File
@@ -4,6 +4,7 @@ import enum
import os
import platform
import random
import sys
from datetime import timedelta
from platform import uname
from typing import TYPE_CHECKING, NamedTuple, Optional, Union
@@ -164,6 +165,9 @@ class Platform:
def is_out_of_tree(self) -> bool:
return self._enum == PlatformEnum.OOT
def get_max_output_tokens(self, prompt_len: int) -> int:
return sys.maxsize
def is_cuda_alike(self) -> bool:
"""Stateless version of [torch.cuda.is_available][]."""
return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)