[Frontend] Consolidate tokenizer init code (#26276)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-10-06 09:34:52 +00:00
committed by GitHub
parent 77c95f72f7
commit 391612e78b
8 changed files with 46 additions and 70 deletions
+10 -12
View File
@@ -23,7 +23,7 @@ from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingParams
from vllm.tasks import SupportedTask
from vllm.tracing import init_tracer
from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
from vllm.transformers_utils.tokenizer import AnyTokenizer
from vllm.usage.usage_lib import UsageContext
from vllm.utils import Device
from vllm.v1.engine import EngineCoreRequest
@@ -95,18 +95,8 @@ class LLMEngine:
self.dp_group = None
self.should_execute_dummy_batch = False
if self.model_config.skip_tokenizer_init:
self.tokenizer = None
else:
# Tokenizer (+ ensure liveness if running in another process).
self.tokenizer = init_tokenizer_from_configs(
model_config=vllm_config.model_config
)
# Processor (convert Inputs --> EngineCoreRequests)
self.processor = Processor(
vllm_config=vllm_config, tokenizer=self.tokenizer, mm_registry=mm_registry
)
self.processor = Processor(vllm_config, mm_registry=mm_registry)
# OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
self.output_processor = OutputProcessor(
@@ -214,6 +204,14 @@ class LLMEngine:
def validate_outputs(cls, outputs, output_type):
return outputs
@property
def tokenizer(self) -> Optional[AnyTokenizer]:
return self.processor.tokenizer
@tokenizer.setter
def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
self.processor.tokenizer = tokenizer
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
return self.engine_core.get_supported_tasks()