Support encoder-only models without KV-Cache (#21270)

Signed-off-by: Max de Bayser <maxdebayser@gmail.com>
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Co-authored-by: Russell Bryant <rbryant@redhat.com>
This commit is contained in:
Maximilien de Bayser
2025-07-26 21:09:52 +08:00
committed by GitHub
co-authored by Russell Bryant
parent f27fdfc3ed
commit 1cd6eaba54
17 changed files with 352 additions and 99 deletions
+4
View File
@@ -4,6 +4,7 @@ from contextlib import contextmanager
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.logger import init_logger
@@ -59,6 +60,9 @@ class CPUModelRunner(GPUModelRunner):
self.scheduler_config,
self.lora_config, self.device)
def get_model(self) -> nn.Module:
return self.model
def warming_up_model(self) -> None:
logger.info("Warming up model for the compilation...")
# Only generate graph for the generic shape