[BUGFIX] Multimodal benchmark with MistralTokenizer (#42965)

Signed-off-by: juliendenize <julien.denize@mistral.ai>
Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com>
This commit is contained in:
Julien Denize
2026-05-28 05:36:24 -07:00
committed by GitHub
parent 19af4e6dd4
commit 02606b0b09
+14 -10
View File
@@ -48,6 +48,7 @@ from vllm.multimodal.image import convert_image_mode
from vllm.tokenizers import TokenizerLike
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm.utils.import_utils import PlaceholderModule
from vllm.utils.mistral import is_mistral_tokenizer
try:
from datasets import load_dataset
@@ -1219,16 +1220,19 @@ class RandomMultiModalDataset(RandomDataset):
)
vocab_size = tokenizer.vocab_size
# Can't use tokenizer.all_special_ids since
# it returns ONLY ids from special_tokens_map.json
# We want to exclude placeholder tokens and all
# tokens that indicate start/end of image as it
# may break prompt replacement logic.
prohibited_tokens = list(
tok_id
for tok_id, token in tokenizer.added_tokens_decoder.items()
if token.special
)
if is_mistral_tokenizer(tokenizer):
prohibited_tokens = tokenizer.all_special_ids
else:
# Can't use tokenizer.all_special_ids since
# it returns ONLY ids from special_tokens_map.json
# We want to exclude placeholder tokens and all
# tokens that indicate start/end of image as it
# may break prompt replacement logic.
prohibited_tokens = list(
tok_id
for tok_id, token in tokenizer.added_tokens_decoder.items()
if token.special
)
all_tokens = np.arange(vocab_size)
allowed_tokens = np.array(list(set(all_tokens) - set(prohibited_tokens)))
logger.debug(