Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+63
-40
@@ -39,27 +39,31 @@ class TpuPlatform(Platform):
|
||||
device_control_env_var: str = "TPU_VISIBLE_CHIPS"
|
||||
simple_compile_backend: str = "openxla"
|
||||
|
||||
supported_quantization: list[str] = [
|
||||
"fp8", "tpu_int8", "compressed-tensors"
|
||||
]
|
||||
supported_quantization: list[str] = ["fp8", "tpu_int8", "compressed-tensors"]
|
||||
|
||||
additional_env_vars: list[str] = [
|
||||
"TPU_CHIPS_PER_HOST_BOUNDS", "TPU_HOST_BOUNDS"
|
||||
]
|
||||
additional_env_vars: list[str] = ["TPU_CHIPS_PER_HOST_BOUNDS", "TPU_HOST_BOUNDS"]
|
||||
|
||||
@classmethod
|
||||
def import_core_kernels(cls) -> None:
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def get_attn_backend_cls(cls, selected_backend: "_Backend", head_size: int,
|
||||
dtype: torch.dtype, kv_cache_dtype: Optional[str],
|
||||
block_size: int, use_v1: bool, use_mla: bool,
|
||||
has_sink, use_sparse) -> str:
|
||||
def get_attn_backend_cls(
|
||||
cls,
|
||||
selected_backend: "_Backend",
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: Optional[str],
|
||||
block_size: int,
|
||||
use_v1: bool,
|
||||
use_mla: bool,
|
||||
has_sink,
|
||||
use_sparse,
|
||||
) -> str:
|
||||
from vllm.attention.backends.registry import _Backend
|
||||
|
||||
if use_sparse:
|
||||
raise NotImplementedError(
|
||||
"Sparse Attention is not supported on TPU.")
|
||||
raise NotImplementedError("Sparse Attention is not supported on TPU.")
|
||||
if selected_backend != _Backend.PALLAS:
|
||||
logger.info("Cannot use %s backend on TPU.", selected_backend)
|
||||
|
||||
@@ -116,34 +120,43 @@ class TpuPlatform(Platform):
|
||||
|
||||
# TPU only supports DYNAMO_ONCE compilation level
|
||||
if compilation_config.level != CompilationLevel.DYNAMO_ONCE:
|
||||
logger.info("[TPU] Forcing DYNAMO_ONCE compilation level, and "
|
||||
"disabling cudagraph.")
|
||||
logger.info(
|
||||
"[TPU] Forcing DYNAMO_ONCE compilation level, and disabling cudagraph."
|
||||
)
|
||||
compilation_config.level = CompilationLevel.DYNAMO_ONCE
|
||||
|
||||
if compilation_config.cudagraph_mode is None or \
|
||||
compilation_config.cudagraph_mode.max_cudagraph_mode() \
|
||||
!= CUDAGraphMode.NONE:
|
||||
logger.info("[TPU] CUDA graph is not supported on TPU, "
|
||||
"disabling cudagraphs.")
|
||||
if (
|
||||
compilation_config.cudagraph_mode is None
|
||||
or compilation_config.cudagraph_mode.max_cudagraph_mode()
|
||||
!= CUDAGraphMode.NONE
|
||||
):
|
||||
logger.info(
|
||||
"[TPU] CUDA graph is not supported on TPU, disabling cudagraphs."
|
||||
)
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
if compilation_config.backend == "":
|
||||
compilation_config.backend = "openxla"
|
||||
|
||||
assert vllm_config.speculative_config is None, \
|
||||
assert vllm_config.speculative_config is None, (
|
||||
"TPU does not support speculative decoding"
|
||||
)
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
if model_config is not None and model_config.dtype in (torch.float16,
|
||||
torch.float32):
|
||||
if model_config is not None and model_config.dtype in (
|
||||
torch.float16,
|
||||
torch.float32,
|
||||
):
|
||||
logger.warning(
|
||||
"The TPU backend currently does not support %s. "
|
||||
"Using bfloat16 instead.", model_config.dtype)
|
||||
"Using bfloat16 instead.",
|
||||
model_config.dtype,
|
||||
)
|
||||
model_config.dtype = torch.bfloat16
|
||||
|
||||
from vllm.v1.attention.backends.pallas import PallasAttentionBackend
|
||||
cache_config.block_size = PallasAttentionBackend.get_page_size(
|
||||
vllm_config) # type: ignore[assignment]
|
||||
|
||||
cache_config.block_size = PallasAttentionBackend.get_page_size(vllm_config) # type: ignore[assignment]
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
@@ -151,24 +164,31 @@ class TpuPlatform(Platform):
|
||||
parallel_config.worker_cls = "vllm.v1.worker.tpu_worker.TPUWorker"
|
||||
|
||||
assert not vllm_config.speculative_config, (
|
||||
"Speculative decoding is not yet supported for TPU backend")
|
||||
"Speculative decoding is not yet supported for TPU backend"
|
||||
)
|
||||
|
||||
if scheduler_config.is_multimodal_model and not \
|
||||
scheduler_config.disable_chunked_mm_input:
|
||||
logger.warning("TPU does not support running Multimodal models"\
|
||||
" without setting `--disable_chunked_mm_input`. " \
|
||||
"Forcing --disable_chunked_mm_input.")
|
||||
if (
|
||||
scheduler_config.is_multimodal_model
|
||||
and not scheduler_config.disable_chunked_mm_input
|
||||
):
|
||||
logger.warning(
|
||||
"TPU does not support running Multimodal models"
|
||||
" without setting `--disable_chunked_mm_input`. "
|
||||
"Forcing --disable_chunked_mm_input."
|
||||
)
|
||||
scheduler_config.disable_chunked_mm_input = True
|
||||
|
||||
if model_config and model_config.use_mla:
|
||||
logger.info(
|
||||
"MLA is enabled on a non-GPU platform; forcing chunked "
|
||||
"prefill and prefix caching to be disabled.")
|
||||
"prefill and prefix caching to be disabled."
|
||||
)
|
||||
vllm_config.scheduler_config.enable_chunked_prefill = False
|
||||
vllm_config.scheduler_config.chunked_prefill_enabled = False
|
||||
vllm_config.scheduler_config.max_num_batched_tokens = max(
|
||||
vllm_config.scheduler_config.max_model_len,
|
||||
DEFAULT_MAX_NUM_BATCHED_TOKENS)
|
||||
DEFAULT_MAX_NUM_BATCHED_TOKENS,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def is_pin_memory_available(cls):
|
||||
@@ -191,13 +211,16 @@ class TpuPlatform(Platform):
|
||||
processed_inputs: ProcessorInputs,
|
||||
) -> None:
|
||||
"""Raises if this request is unsupported on this platform"""
|
||||
if (isinstance(params, SamplingParams)
|
||||
and params.sampling_type == SamplingType.RANDOM_SEED):
|
||||
if (
|
||||
isinstance(params, SamplingParams)
|
||||
and params.sampling_type == SamplingType.RANDOM_SEED
|
||||
):
|
||||
raise ValueError("Torch XLA does not support per-request seed.")
|
||||
|
||||
@classmethod
|
||||
def is_kv_cache_dtype_supported(cls, kv_cache_dtype: str,
|
||||
model_config: "ModelConfig") -> bool:
|
||||
def is_kv_cache_dtype_supported(
|
||||
cls, kv_cache_dtype: str, model_config: "ModelConfig"
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
@@ -210,8 +233,7 @@ class TpuPlatform(Platform):
|
||||
dst_block_indices: torch.Tensor,
|
||||
) -> None:
|
||||
torch.ops.xla.dynamo_set_buffer_donor_(dst_cache, True)
|
||||
dst_cache[dst_block_indices] = src_cache[src_block_indices].to(
|
||||
dst_cache.device)
|
||||
dst_cache[dst_block_indices] = src_cache[src_block_indices].to(dst_cache.device)
|
||||
|
||||
@classmethod
|
||||
@torch.compile(backend="openxla")
|
||||
@@ -222,7 +244,7 @@ class TpuPlatform(Platform):
|
||||
src_block_indices: torch.Tensor,
|
||||
dst_block_indices: torch.Tensor,
|
||||
) -> None:
|
||||
""" tpu blocks to cpu blocks"""
|
||||
"""tpu blocks to cpu blocks"""
|
||||
torch.ops.xla.dynamo_set_buffer_donor_(src_cache, True)
|
||||
dst_cache[dst_block_indices] = src_cache[src_block_indices].cpu()
|
||||
|
||||
@@ -233,6 +255,7 @@ class TpuPlatform(Platform):
|
||||
|
||||
try:
|
||||
from tpu_commons.platforms import TpuPlatform as TpuCommonsPlatform
|
||||
|
||||
TpuPlatform = TpuCommonsPlatform # type: ignore
|
||||
USE_TPU_COMMONS = True
|
||||
except ImportError:
|
||||
|
||||
Reference in New Issue
Block a user