[TPU] Fix TPU SMEM OOM by Pallas paged attention kernel (#9350)

This commit is contained in:
Woosuk Kwon
2024-10-14 15:02:06 -07:00
committed by GitHub
parent fd47e57f4b
commit 473e7b3606
2 changed files with 88 additions and 20 deletions
+9
View File
@@ -123,6 +123,15 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
)
self.cached_step_outputs: List[torch.Tensor] = []
smem_size = 512 * 1024
block_table_size = 4 * self.block_tables.size
if block_table_size >= smem_size:
logger.warning(
"The max_model_len (%d) is too large. This may degrade the "
"performance due to the insufficient smem size. Consider "
"setting --max-model-len to a smaller value.",
self.model_config.max_model_len)
def load_model(self) -> None:
self.device = self.device_config.device