[Misc] Refactor tokenizer interface (#29693)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-11-29 04:02:21 -08:00
committed by GitHub
parent f223ed4181
commit 34a984274e
119 changed files with 752 additions and 821 deletions
+9 -7
View File
@@ -26,9 +26,10 @@ from vllm.plugins.io_processors import get_io_processor
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingParams
from vllm.tasks import SupportedTask
from vllm.tokenizers import TokenizerLike
from vllm.tracing import init_tracer
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
from vllm.usage.usage_lib import UsageContext
from vllm.utils.async_utils import cancel_task_threadsafe
from vllm.utils.collection_utils import as_list
@@ -120,9 +121,10 @@ class AsyncLLM(EngineClient):
)
# OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
stream_interval = self.vllm_config.scheduler_config.stream_interval
self.output_processor = OutputProcessor(
self.tokenizer, log_stats=self.log_stats, stream_interval=stream_interval
self.tokenizer,
log_stats=self.log_stats,
stream_interval=self.vllm_config.scheduler_config.stream_interval,
)
endpoint = self.observability_config.otlp_traces_endpoint
if endpoint is not None:
@@ -703,17 +705,17 @@ class AsyncLLM(EngineClient):
raise EngineGenerateError() from e
@property
def tokenizer(self) -> AnyTokenizer | None:
def tokenizer(self) -> TokenizerLike | None:
return self.input_processor.tokenizer
@tokenizer.setter
def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:
def tokenizer(self, tokenizer: TokenizerLike | None) -> None:
self.input_processor.tokenizer = tokenizer
async def get_tokenizer(self) -> AnyTokenizer:
async def get_tokenizer(self) -> TokenizerLike:
if self.tokenizer is None:
raise ValueError(
"Unable to get tokenizer because skip_tokenizer_init is True"
"Unable to get tokenizer because `skip_tokenizer_init=True`"
)
return self.tokenizer
+3 -3
View File
@@ -10,7 +10,7 @@ from transformers import PreTrainedTokenizerFast
from vllm.logger import init_logger
from vllm.transformers_utils.detokenizer_utils import (
AnyTokenizer,
TokenizerLike,
convert_prompt_ids_to_tokens,
detokenize_incrementally,
)
@@ -45,7 +45,7 @@ class IncrementalDetokenizer:
@classmethod
def from_new_request(
cls,
tokenizer: AnyTokenizer | None,
tokenizer: TokenizerLike | None,
request: EngineCoreRequest,
) -> "IncrementalDetokenizer":
assert request.sampling_params is not None
@@ -256,7 +256,7 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
def __init__(self, tokenizer: AnyTokenizer, request: EngineCoreRequest):
def __init__(self, tokenizer: TokenizerLike, request: EngineCoreRequest):
super().__init__(request)
self.tokenizer = tokenizer
+4 -5
View File
@@ -19,8 +19,7 @@ from vllm.multimodal.processing import EncDecMultiModalProcessor
from vllm.multimodal.utils import argsort_mm_positions
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingParams
from vllm.transformers_utils.tokenizer import AnyTokenizer
from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
from vllm.tokenizers import MistralTokenizer, TokenizerLike
from vllm.utils import length_from_prompt_token_ids_or_embeds
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.metrics.stats import MultiModalCacheStats
@@ -40,7 +39,7 @@ class InputProcessor:
def __init__(
self,
vllm_config: VllmConfig,
tokenizer: AnyTokenizer | None,
tokenizer: TokenizerLike | None,
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
) -> None:
self.vllm_config = vllm_config
@@ -62,11 +61,11 @@ class InputProcessor:
)
@property
def tokenizer(self) -> AnyTokenizer | None:
def tokenizer(self) -> TokenizerLike | None:
return self.input_preprocessor.tokenizer
@tokenizer.setter
def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:
def tokenizer(self, tokenizer: TokenizerLike | None) -> None:
self.input_preprocessor.tokenizer = tokenizer
def _validate_logprobs(
+9 -7
View File
@@ -23,8 +23,9 @@ from vllm.plugins.io_processors import get_io_processor
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingParams
from vllm.tasks import SupportedTask
from vllm.tokenizers import TokenizerLike
from vllm.tracing import init_tracer
from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
from vllm.usage.usage_lib import UsageContext
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.core_client import EngineCoreClient
@@ -95,9 +96,10 @@ class LLMEngine:
)
# OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
stream_interval = self.vllm_config.scheduler_config.stream_interval
self.output_processor = OutputProcessor(
self.tokenizer, log_stats=self.log_stats, stream_interval=stream_interval
self.tokenizer,
log_stats=self.log_stats,
stream_interval=self.vllm_config.scheduler_config.stream_interval,
)
endpoint = self.observability_config.otlp_traces_endpoint
if endpoint is not None:
@@ -350,17 +352,17 @@ class LLMEngine:
return get_metrics_snapshot()
@property
def tokenizer(self) -> AnyTokenizer | None:
def tokenizer(self) -> TokenizerLike | None:
return self.input_processor.tokenizer
@tokenizer.setter
def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:
def tokenizer(self, tokenizer: TokenizerLike | None) -> None:
self.input_processor.tokenizer = tokenizer
def get_tokenizer(self) -> AnyTokenizer:
def get_tokenizer(self) -> TokenizerLike:
if self.tokenizer is None:
raise ValueError(
"Unable to get tokenizer because skip_tokenizer_init is True"
"Unable to get tokenizer because `skip_tokenizer_init=True`"
)
return self.tokenizer
+3 -3
View File
@@ -13,7 +13,7 @@ from vllm.logprobs import (
create_sample_logprobs,
)
from vllm.transformers_utils.detokenizer_utils import (
AnyTokenizer,
TokenizerLike,
convert_ids_list_to_tokens,
)
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
@@ -28,7 +28,7 @@ NONES = itertools.repeat(None)
class LogprobsProcessor:
# Tokenizer for this request,
# None if detokenization is disabled.
tokenizer: AnyTokenizer | None
tokenizer: TokenizerLike | None
# Logprobs for this request
logprobs: SampleLogprobs | None
@@ -40,7 +40,7 @@ class LogprobsProcessor:
@classmethod
def from_new_request(
cls,
tokenizer: AnyTokenizer | None,
tokenizer: TokenizerLike | None,
request: EngineCoreRequest,
) -> "LogprobsProcessor":
sampling_params = request.sampling_params
+6 -3
View File
@@ -15,8 +15,8 @@ from vllm.outputs import (
RequestOutput,
)
from vllm.sampling_params import RequestOutputKind
from vllm.tokenizers import TokenizerLike
from vllm.tracing import SpanAttributes, SpanKind, Tracer, extract_trace_context
from vllm.transformers_utils.tokenizer import AnyTokenizer
from vllm.utils import length_from_prompt_token_ids_or_embeds
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
from vllm.v1.engine.detokenizer import IncrementalDetokenizer
@@ -139,7 +139,7 @@ class RequestState:
@classmethod
def from_new_request(
cls,
tokenizer: AnyTokenizer,
tokenizer: TokenizerLike | None,
request: EngineCoreRequest,
prompt: str | None,
parent_req: ParentRequest | None,
@@ -341,7 +341,10 @@ class OutputProcessor:
"""Process EngineCoreOutputs into RequestOutputs."""
def __init__(
self, tokenizer: AnyTokenizer, log_stats: bool, stream_interval: int = 1
self,
tokenizer: TokenizerLike | None,
log_stats: bool,
stream_interval: int = 1,
):
self.log_stats = log_stats
self.tokenizer = tokenizer