[Misc] Refactor tokenizer interface (#29693)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -26,9 +26,10 @@ from vllm.plugins.io_processors import get_io_processor
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tracing import init_tracer
|
||||
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
|
||||
from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils.async_utils import cancel_task_threadsafe
|
||||
from vllm.utils.collection_utils import as_list
|
||||
@@ -120,9 +121,10 @@ class AsyncLLM(EngineClient):
|
||||
)
|
||||
|
||||
# OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
|
||||
stream_interval = self.vllm_config.scheduler_config.stream_interval
|
||||
self.output_processor = OutputProcessor(
|
||||
self.tokenizer, log_stats=self.log_stats, stream_interval=stream_interval
|
||||
self.tokenizer,
|
||||
log_stats=self.log_stats,
|
||||
stream_interval=self.vllm_config.scheduler_config.stream_interval,
|
||||
)
|
||||
endpoint = self.observability_config.otlp_traces_endpoint
|
||||
if endpoint is not None:
|
||||
@@ -703,17 +705,17 @@ class AsyncLLM(EngineClient):
|
||||
raise EngineGenerateError() from e
|
||||
|
||||
@property
|
||||
def tokenizer(self) -> AnyTokenizer | None:
|
||||
def tokenizer(self) -> TokenizerLike | None:
|
||||
return self.input_processor.tokenizer
|
||||
|
||||
@tokenizer.setter
|
||||
def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:
|
||||
def tokenizer(self, tokenizer: TokenizerLike | None) -> None:
|
||||
self.input_processor.tokenizer = tokenizer
|
||||
|
||||
async def get_tokenizer(self) -> AnyTokenizer:
|
||||
async def get_tokenizer(self) -> TokenizerLike:
|
||||
if self.tokenizer is None:
|
||||
raise ValueError(
|
||||
"Unable to get tokenizer because skip_tokenizer_init is True"
|
||||
"Unable to get tokenizer because `skip_tokenizer_init=True`"
|
||||
)
|
||||
|
||||
return self.tokenizer
|
||||
|
||||
@@ -10,7 +10,7 @@ from transformers import PreTrainedTokenizerFast
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.detokenizer_utils import (
|
||||
AnyTokenizer,
|
||||
TokenizerLike,
|
||||
convert_prompt_ids_to_tokens,
|
||||
detokenize_incrementally,
|
||||
)
|
||||
@@ -45,7 +45,7 @@ class IncrementalDetokenizer:
|
||||
@classmethod
|
||||
def from_new_request(
|
||||
cls,
|
||||
tokenizer: AnyTokenizer | None,
|
||||
tokenizer: TokenizerLike | None,
|
||||
request: EngineCoreRequest,
|
||||
) -> "IncrementalDetokenizer":
|
||||
assert request.sampling_params is not None
|
||||
@@ -256,7 +256,7 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
|
||||
|
||||
class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
|
||||
def __init__(self, tokenizer: AnyTokenizer, request: EngineCoreRequest):
|
||||
def __init__(self, tokenizer: TokenizerLike, request: EngineCoreRequest):
|
||||
super().__init__(request)
|
||||
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
@@ -19,8 +19,7 @@ from vllm.multimodal.processing import EncDecMultiModalProcessor
|
||||
from vllm.multimodal.utils import argsort_mm_positions
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.tokenizers import MistralTokenizer, TokenizerLike
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.metrics.stats import MultiModalCacheStats
|
||||
@@ -40,7 +39,7 @@ class InputProcessor:
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
tokenizer: AnyTokenizer | None,
|
||||
tokenizer: TokenizerLike | None,
|
||||
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
|
||||
) -> None:
|
||||
self.vllm_config = vllm_config
|
||||
@@ -62,11 +61,11 @@ class InputProcessor:
|
||||
)
|
||||
|
||||
@property
|
||||
def tokenizer(self) -> AnyTokenizer | None:
|
||||
def tokenizer(self) -> TokenizerLike | None:
|
||||
return self.input_preprocessor.tokenizer
|
||||
|
||||
@tokenizer.setter
|
||||
def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:
|
||||
def tokenizer(self, tokenizer: TokenizerLike | None) -> None:
|
||||
self.input_preprocessor.tokenizer = tokenizer
|
||||
|
||||
def _validate_logprobs(
|
||||
|
||||
@@ -23,8 +23,9 @@ from vllm.plugins.io_processors import get_io_processor
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tracing import init_tracer
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
|
||||
from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.core_client import EngineCoreClient
|
||||
@@ -95,9 +96,10 @@ class LLMEngine:
|
||||
)
|
||||
|
||||
# OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
|
||||
stream_interval = self.vllm_config.scheduler_config.stream_interval
|
||||
self.output_processor = OutputProcessor(
|
||||
self.tokenizer, log_stats=self.log_stats, stream_interval=stream_interval
|
||||
self.tokenizer,
|
||||
log_stats=self.log_stats,
|
||||
stream_interval=self.vllm_config.scheduler_config.stream_interval,
|
||||
)
|
||||
endpoint = self.observability_config.otlp_traces_endpoint
|
||||
if endpoint is not None:
|
||||
@@ -350,17 +352,17 @@ class LLMEngine:
|
||||
return get_metrics_snapshot()
|
||||
|
||||
@property
|
||||
def tokenizer(self) -> AnyTokenizer | None:
|
||||
def tokenizer(self) -> TokenizerLike | None:
|
||||
return self.input_processor.tokenizer
|
||||
|
||||
@tokenizer.setter
|
||||
def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:
|
||||
def tokenizer(self, tokenizer: TokenizerLike | None) -> None:
|
||||
self.input_processor.tokenizer = tokenizer
|
||||
|
||||
def get_tokenizer(self) -> AnyTokenizer:
|
||||
def get_tokenizer(self) -> TokenizerLike:
|
||||
if self.tokenizer is None:
|
||||
raise ValueError(
|
||||
"Unable to get tokenizer because skip_tokenizer_init is True"
|
||||
"Unable to get tokenizer because `skip_tokenizer_init=True`"
|
||||
)
|
||||
|
||||
return self.tokenizer
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.logprobs import (
|
||||
create_sample_logprobs,
|
||||
)
|
||||
from vllm.transformers_utils.detokenizer_utils import (
|
||||
AnyTokenizer,
|
||||
TokenizerLike,
|
||||
convert_ids_list_to_tokens,
|
||||
)
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
|
||||
@@ -28,7 +28,7 @@ NONES = itertools.repeat(None)
|
||||
class LogprobsProcessor:
|
||||
# Tokenizer for this request,
|
||||
# None if detokenization is disabled.
|
||||
tokenizer: AnyTokenizer | None
|
||||
tokenizer: TokenizerLike | None
|
||||
|
||||
# Logprobs for this request
|
||||
logprobs: SampleLogprobs | None
|
||||
@@ -40,7 +40,7 @@ class LogprobsProcessor:
|
||||
@classmethod
|
||||
def from_new_request(
|
||||
cls,
|
||||
tokenizer: AnyTokenizer | None,
|
||||
tokenizer: TokenizerLike | None,
|
||||
request: EngineCoreRequest,
|
||||
) -> "LogprobsProcessor":
|
||||
sampling_params = request.sampling_params
|
||||
|
||||
@@ -15,8 +15,8 @@ from vllm.outputs import (
|
||||
RequestOutput,
|
||||
)
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tracing import SpanAttributes, SpanKind, Tracer, extract_trace_context
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
|
||||
from vllm.v1.engine.detokenizer import IncrementalDetokenizer
|
||||
@@ -139,7 +139,7 @@ class RequestState:
|
||||
@classmethod
|
||||
def from_new_request(
|
||||
cls,
|
||||
tokenizer: AnyTokenizer,
|
||||
tokenizer: TokenizerLike | None,
|
||||
request: EngineCoreRequest,
|
||||
prompt: str | None,
|
||||
parent_req: ParentRequest | None,
|
||||
@@ -341,7 +341,10 @@ class OutputProcessor:
|
||||
"""Process EngineCoreOutputs into RequestOutputs."""
|
||||
|
||||
def __init__(
|
||||
self, tokenizer: AnyTokenizer, log_stats: bool, stream_interval: int = 1
|
||||
self,
|
||||
tokenizer: TokenizerLike | None,
|
||||
log_stats: bool,
|
||||
stream_interval: int = 1,
|
||||
):
|
||||
self.log_stats = log_stats
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
Reference in New Issue
Block a user