[TPU] Increase block size and reset block shapes (#16458)

This commit is contained in:
Jevin Jiang
2025-05-06 13:55:04 -04:00
committed by GitHub
parent 6115b11582
commit 621ca2c0ab
5 changed files with 35 additions and 11 deletions
+6 -4
View File
@@ -76,9 +76,9 @@ class TpuPlatform(Platform):
from vllm.config import CompilationLevel
cache_config = vllm_config.cache_config
# For v0, the default block size is 16.
if cache_config and cache_config.block_size is None:
cache_config.block_size = 16
compilation_config = vllm_config.compilation_config
# TPU only supports DYNAMO_ONCE compilation level
@@ -101,16 +101,18 @@ class TpuPlatform(Platform):
if envs.VLLM_USE_V1:
from vllm.v1.attention.backends.pallas import (
PallasAttentionBackend)
cache_config.block_size = PallasAttentionBackend.get_page_size(
vllm_config)
min_page_size = PallasAttentionBackend.get_min_page_size(
vllm_config)
if min_page_size > vllm_config.cache_config.block_size:
if min_page_size > cache_config.block_size:
logger.warning(
"Increase the page size from %s to %s to make sure there's"
"no SMEM OOM",
vllm_config.cache_config.block_size,
cache_config.block_size,
min_page_size,
)
vllm_config.cache_config.block_size = min_page_size
cache_config.block_size = min_page_size
parallel_config = vllm_config.parallel_config
scheduler_config = vllm_config.scheduler_config
+7
View File
@@ -707,6 +707,13 @@ def cdiv(a: int, b: int) -> int:
return -(a // -b)
def next_power_of_2(n) -> int:
"""The next power of 2 (inclusive)"""
if n < 1:
return 1
return 1 << (n - 1).bit_length()
def round_up(x: int, y: int) -> int:
return ((x + y - 1) // y) * y
+15 -1
View File
@@ -12,7 +12,7 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
from vllm.attention.backends.utils import CommonAttentionState
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.utils import cdiv
from vllm.utils import cdiv, next_power_of_2
logger = init_logger(__name__)
@@ -65,6 +65,20 @@ class PallasAttentionBackend(AttentionBackend):
min_page_size = 1 << (min_page_size - 1).bit_length()
return min_page_size
# TPU has limited SREGs (scalar registers), if page_size is too small, we
# can spill SREGs easily which leads to bad performance. The strategy we
# apply here is trying to split max-model-len to 16 pages which make the
# spill less likely. Meanwhile we make sure the page size is in [16, 256].
@staticmethod
def get_page_size(vllm_config: VllmConfig) -> int:
page_size = next_power_of_2(
vllm_config.model_config.max_model_len) // 16
if page_size <= 16:
return 16
if page_size >= 256:
return 256
return page_size
@dataclass
class PallasMetadata: