[Bugfix] Support MistralCommonBackend tokenizers in structured output (#52720)
Signed-off-by: Thanh Phan <thanhphantuan1110@gmail.com>
This commit is contained in:
@@ -17,6 +17,7 @@ from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputGrammar,
|
||||
)
|
||||
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
|
||||
from vllm.v1.structured_output.utils import maybe_wrap_mistral_common_tokenizer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import numpy as np
|
||||
@@ -75,8 +76,8 @@ class StructuredOutputManager:
|
||||
# of CPUs.
|
||||
max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2)
|
||||
self.executor = ThreadPoolExecutor(max_workers=max_workers)
|
||||
self.tokenizer = cached_tokenizer_from_config(
|
||||
model_config=self.vllm_config.model_config
|
||||
self.tokenizer = maybe_wrap_mistral_common_tokenizer(
|
||||
cached_tokenizer_from_config(model_config=self.vllm_config.model_config)
|
||||
)
|
||||
reasoning_parser_plugin = (
|
||||
self.vllm_config.structured_outputs_config.reasoning_parser_plugin
|
||||
|
||||
@@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, TypeVar
|
||||
import regex as re
|
||||
import torch
|
||||
from cachetools import LRUCache
|
||||
from transformers import MistralCommonBackend
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
@@ -305,6 +306,18 @@ re_llama_byte_token = re.compile(r"^<0x[0-9A-F]{2}>$")
|
||||
re_replacement_seq = re.compile(r"^.{0,6}�+.{0,6}$")
|
||||
|
||||
|
||||
def maybe_wrap_mistral_common_tokenizer(tokenizer: TokenizerLike) -> TokenizerLike:
|
||||
"""The grammar backends cannot consume a `MistralCommonBackend` directly."""
|
||||
if not isinstance(tokenizer, MistralCommonBackend):
|
||||
return tokenizer
|
||||
|
||||
# Deferred: `vllm.tokenizers.mistral` pulls in a large dependency tree, and
|
||||
# this module is imported by `vllm.v1.worker.gpu_model_runner`.
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
return MistralTokenizer(tokenizer)
|
||||
|
||||
|
||||
def _reduced_vocabulary(tokenizer: TokenizerLike) -> dict[bytes, list[int]]:
|
||||
"""Create a map from vocabulary tokens to lists of equivalent token ids.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user