[DisaggEverything] Tokens in<>out /generate endpoint (#24261)
Signed-off-by: NickLucche <nlucches@redhat.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
d54a18a47e
commit
6f1e7f7226
@@ -566,6 +566,7 @@ class EngineArgs:
|
||||
kv_offloading_backend: KVOffloadingBackend | None = (
|
||||
CacheConfig.kv_offloading_backend
|
||||
)
|
||||
tokens_only: bool = False
|
||||
|
||||
def __post_init__(self):
|
||||
# support `EngineArgs(compilation_config={...})`
|
||||
@@ -1495,6 +1496,10 @@ class EngineArgs:
|
||||
else ParallelConfig.data_parallel_rpc_port
|
||||
)
|
||||
|
||||
if self.tokens_only and not model_config.skip_tokenizer_init:
|
||||
model_config.skip_tokenizer_init = True
|
||||
logger.info("Skipping tokenizer initialization for tokens-only mode.")
|
||||
|
||||
# Forward the deprecated CLI args to the EPLB config.
|
||||
if self.num_redundant_experts is not None:
|
||||
self.eplb_config.num_redundant_experts = self.num_redundant_experts
|
||||
|
||||
Reference in New Issue
Block a user