[DisaggEverything] Tokens in<>out /generate endpoint (#24261)

Signed-off-by: NickLucche <nlucches@redhat.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Nicolò Lucchesi
2025-11-14 09:58:01 -07:00
committed by GitHub
co-authored by Harry Mellor
parent d54a18a47e
commit 6f1e7f7226
12 changed files with 822 additions and 9 deletions
+5
View File
@@ -566,6 +566,7 @@ class EngineArgs:
kv_offloading_backend: KVOffloadingBackend | None = (
CacheConfig.kv_offloading_backend
)
tokens_only: bool = False
def __post_init__(self):
# support `EngineArgs(compilation_config={...})`
@@ -1495,6 +1496,10 @@ class EngineArgs:
else ParallelConfig.data_parallel_rpc_port
)
if self.tokens_only and not model_config.skip_tokenizer_init:
model_config.skip_tokenizer_init = True
logger.info("Skipping tokenizer initialization for tokens-only mode.")
# Forward the deprecated CLI args to the EPLB config.
if self.num_redundant_experts is not None:
self.eplb_config.num_redundant_experts = self.num_redundant_experts