Register parsed config classes before tokenizer init (#40299)
Signed-off-by: Bortlesboat <bortstheboat@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
9d808e2309
commit
a9a8a32dcd
@@ -1,15 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from transformers import AutoConfig
|
||||
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.registry import (
|
||||
TokenizerRegistry,
|
||||
cached_get_tokenizer,
|
||||
cached_resolve_tokenizer_args,
|
||||
cached_tokenizer_from_config,
|
||||
get_tokenizer,
|
||||
resolve_tokenizer_args,
|
||||
)
|
||||
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeConfig
|
||||
|
||||
|
||||
class TestTokenizer(TokenizerLike):
|
||||
@@ -75,3 +84,58 @@ def test_customized_tokenizer():
|
||||
assert tokenizer.bos_token_id == 0
|
||||
assert tokenizer.eos_token_id == 1
|
||||
assert tokenizer.pad_token_id == 2
|
||||
|
||||
|
||||
def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path):
|
||||
(tmp_path / "config.json").write_text(
|
||||
json.dumps({"model_type": "qwen3_5_moe"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
model_config = SimpleNamespace(
|
||||
skip_tokenizer_init=False,
|
||||
tokenizer=str(tmp_path),
|
||||
runner_type="generate",
|
||||
tokenizer_mode="hf",
|
||||
tokenizer_revision=None,
|
||||
trust_remote_code=True,
|
||||
hf_config=Qwen3_5MoeConfig(),
|
||||
)
|
||||
|
||||
registered_config = CONFIG_MAPPING._extra_content.pop("qwen3_5_moe", None)
|
||||
cached_get_tokenizer.cache_clear()
|
||||
cached_resolve_tokenizer_args.cache_clear()
|
||||
|
||||
try:
|
||||
|
||||
def fake_from_pretrained(path_or_repo_id: str, *args, **kwargs):
|
||||
loaded_config = AutoConfig.from_pretrained(
|
||||
path_or_repo_id,
|
||||
trust_remote_code=False,
|
||||
)
|
||||
assert isinstance(loaded_config, Qwen3_5MoeConfig)
|
||||
return SimpleNamespace(is_fast=True)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.tokenizers.registry.logger.debug_once",
|
||||
lambda *args, **kwargs: None,
|
||||
),
|
||||
patch(
|
||||
"vllm.tokenizers.hf.AutoTokenizer.from_pretrained",
|
||||
side_effect=fake_from_pretrained,
|
||||
),
|
||||
patch(
|
||||
"vllm.tokenizers.hf.get_cached_tokenizer",
|
||||
side_effect=lambda tokenizer: tokenizer,
|
||||
),
|
||||
):
|
||||
tokenizer = cached_tokenizer_from_config(model_config)
|
||||
|
||||
assert tokenizer.is_fast is True
|
||||
finally:
|
||||
cached_get_tokenizer.cache_clear()
|
||||
cached_resolve_tokenizer_args.cache_clear()
|
||||
CONFIG_MAPPING._extra_content.pop("qwen3_5_moe", None)
|
||||
if registered_config is not None:
|
||||
CONFIG_MAPPING._extra_content["qwen3_5_moe"] = registered_config
|
||||
|
||||
Reference in New Issue
Block a user