From 9e0b6fcf77b57401a08224df89f893459ceee375 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:42:35 +0100 Subject: [PATCH] [CI] Bump Transformers version to 5.18.0 (#59621) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- requirements/common.txt | 2 +- requirements/test/cpu.txt | 2 +- requirements/test/cuda.in | 2 +- requirements/test/cuda.txt | 2 +- requirements/test/nightly-torch.txt | 2 +- requirements/test/rocm.in | 2 +- requirements/test/rocm.txt | 2 +- requirements/test/xpu.in | 2 +- requirements/test/xpu.txt | 2 +- tests/models/registry.py | 6 +- vllm/config/model.py | 2 + vllm/config/speculative.py | 2 +- vllm/model_executor/models/lightonocr.py | 13 ++ vllm/model_executor/models/longcat_flash.py | 166 ++---------------- .../models/longcat_flash_mtp.py | 13 +- .../models/longcat_flash_ngram.py | 22 +-- vllm/models/hy_v4/nvidia/attention.py | 8 +- vllm/models/qwen4_exp/amd/model.py | 10 +- vllm/models/qwen4_exp/nvidia/model.py | 10 +- vllm/transformers_utils/config.py | 22 +++ .../model_arch_config_convertor.py | 8 +- vllm/v1/worker/gpu/attn_utils.py | 5 +- 22 files changed, 104 insertions(+), 201 deletions(-) diff --git a/requirements/common.txt b/requirements/common.txt index 6e0b0db783..433494056d 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -7,7 +7,7 @@ requests >= 2.33.0 tqdm blake3 py-cpuinfo -transformers >= 5.16.1, < 5.18.0 +transformers >= 5.16.1, < 5.19.0 huggingface_hub >= 1.31.0 # Required for the `huggingface_hub.utils.httpx` re-export tokenizers >= 0.23.2 # Required for fast incremental detokenization. safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611 diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index 926a5ceb1c..91fac06966 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -1228,7 +1228,7 @@ traitlets==5.16.1 # via # jupyter-client # jupyter-core -transformers==5.17.0 +transformers==5.18.0 # via # -r requirements/test/../common.txt # -r requirements/test/cuda.in diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index cb5ceb248e..4732f3a505 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -40,7 +40,7 @@ datamodel_code_generator>=0.64.0 # required for minicpm3 test nltk>=3.10.3 # pin transitive rouge-score/lm-eval dep for known GHSAs lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test -transformers==5.17.0 +transformers==5.18.0 schemathesis>=4.0.0 # Required for openai schema test. # quantization buildkite-test-collector==0.1.9 diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index 8f39bb900e..ad185844be 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -1335,7 +1335,7 @@ traitlets==5.16.1 # via # jupyter-client # jupyter-core -transformers==5.17.0 +transformers==5.18.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/nightly-torch.txt b/requirements/test/nightly-torch.txt index 256683967e..a229116859 100644 --- a/requirements/test/nightly-torch.txt +++ b/requirements/test/nightly-torch.txt @@ -29,7 +29,7 @@ opencv-python-headless >= 4.13.0 # required for video test datamodel_code_generator>=0.64.0 # required for minicpm3 test lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test -transformers==5.17.0 +transformers==5.18.0 schemathesis>=4.0.0 # Required for openai schema test. # quantization buildkite-test-collector==0.1.9 diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index e66343d47b..753fad0810 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -36,7 +36,7 @@ datamodel_code_generator>=0.64.0 # required for minicpm3 test nltk>=3.10.3 # pin transitive rouge-score/lm-eval dep for known GHSAs lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test -transformers==5.17.0 +transformers==5.18.0 schemathesis>=4.0.0 # Required for openai schema test # quantization bitsandbytes==0.50.0 diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 0df6f9c82a..e6c137ebf5 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -1282,7 +1282,7 @@ traitlets==5.16.1 # via # jupyter-client # jupyter-core -transformers==5.17.0 +transformers==5.18.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/xpu.in b/requirements/test/xpu.in index c39e8350dd..4b3d68ddfc 100644 --- a/requirements/test/xpu.in +++ b/requirements/test/xpu.in @@ -20,7 +20,7 @@ nltk>=3.10.3 # pin transitive rouge-score/lm-eval dep for known GHSAs modelscope<1.38 runai-model-streamer[s3,gcs,azure]>=0.15.7 tensorizer==2.10.1 -transformers==5.17.0 +transformers==5.18.0 # --- Audio Processing --- av diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index 33b89a85fc..19cd271017 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -1134,7 +1134,7 @@ traitlets==5.16.1 # via # jupyter-client # jupyter-core -transformers==5.17.0 +transformers==5.18.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/tests/models/registry.py b/tests/models/registry.py index 80ca856615..b764cad595 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -380,12 +380,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "Llama4ForCausalLM": _HfExamplesInfo( "meta-llama/Llama-4-Scout-17B-16E-Instruct", ), - "LongcatFlashForCausalLM": _HfExamplesInfo( - "meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True - ), + "LongcatFlashForCausalLM": _HfExamplesInfo("meituan-longcat/LongCat-Flash-Chat"), "LongcatFlashNgramForCausalLM": _HfExamplesInfo( "meituan-longcat/LongCat-Flash-Lite", - trust_remote_code=True, # Shrink the ~62GB n-gram tables (ngram_vocab_size_ratio * vocab_size) # so the dummy-weight init test fits in CI memory. hf_overrides={"ngram_vocab_size_ratio": 1}, @@ -1752,7 +1749,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { ), "LongCatFlashMTPModel": _HfExamplesInfo( "meituan-longcat/LongCat-Flash-Chat", - trust_remote_code=True, speculative_model="meituan-longcat/LongCat-Flash-Chat", ), "MiMoMTPModel": _HfExamplesInfo( diff --git a/vllm/config/model.py b/vllm/config/model.py index 75ca774fa1..3e481d8c91 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -111,6 +111,8 @@ LayerBlockType = Literal["attention", "linear_attention", "mamba"] _ATTENTION_LAYER_TYPES = frozenset( { "full_attention", + "indexed_attention", + # TODO: Delete below once Transformers 5.18.0 is the minimum required version. "deepseek_sparse_attention", "qwen_sparse_attention", } diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index ff4f76153d..47ffb3b7f7 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -960,7 +960,7 @@ class SpeculativeConfig: "architectures": [architecture], } ) - if hf_config.model_type in ("longcat_flash", "longcat_flash_ngram"): + if hf_config.model_type == "longcat_flash": hf_config.model_type = "longcat_flash_mtp" n_predict = getattr(hf_config, "num_nextn_predict_layers", 1) hf_config.update( diff --git a/vllm/model_executor/models/lightonocr.py b/vllm/model_executor/models/lightonocr.py index bc0292ed55..d89fa53299 100644 --- a/vllm/model_executor/models/lightonocr.py +++ b/vllm/model_executor/models/lightonocr.py @@ -50,6 +50,19 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessing def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) + def _call_hf_processor( + self, + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], + ) -> BatchFeature: + # The checkpoint sets `do_pad: null`, which disables padding since + # Transformers v5.18, so images of different sizes cannot be stacked. + return self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**hf_kwargs), + hf_data, + {**hf_kwargs, "do_pad": True}, + ) + def _postprocess_hf_mm_data( self, mm_data: Mapping[str, object], diff --git a/vllm/model_executor/models/longcat_flash.py b/vllm/model_executor/models/longcat_flash.py index 0c9802a8b3..50962e9728 100644 --- a/vllm/model_executor/models/longcat_flash.py +++ b/vllm/model_executor/models/longcat_flash.py @@ -39,7 +39,7 @@ from itertools import islice import torch from torch import nn -from transformers import PreTrainedConfig +from transformers import LongcatFlashConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig @@ -80,121 +80,6 @@ from .utils import ( logger = init_logger(__name__) -class FlashConfig(PreTrainedConfig): - """Flash model configuration.""" - - moe_intermediate_size: int - - model_type = "longcat_flash" - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - vocab_size=131072, - hidden_size=4096, - intermediate_size=8192, - num_layers=28, - num_hidden_layers=None, - num_attention_heads=96, - num_key_value_heads=128, - ep_size=1, - kv_lora_rank=512, - q_lora_rank=1536, - qk_rope_head_dim=64, - v_head_dim=128, - qk_nope_head_dim=128, - num_experts_per_tok=None, - norm_topk_prob=False, - max_position_embeddings=8192, - initializer_range=0.02, - rms_norm_eps=1e-05, - use_cache=True, - pad_token_id=None, - bos_token_id=100000, - eos_token_id=100001, - pretraining_tp=1, - tie_word_embeddings=False, - rope_parameters=None, - attention_bias=False, - attention_dropout=0.0, - mla_scale_q_lora=False, - mla_scale_kv_lora=False, - dtype="bfloat16", - params_dtype="bfloat16", - router_dtype="float32", - router_bias=False, - topk_method=None, - routed_scaling_factor=1.0, - zero_expert_num=0, - zero_expert_type=None, - nextn_use_scmoe=False, - **kwargs, - ): - super().__init__( - pad_token_id=pad_token_id, - bos_token_id=bos_token_id, - eos_token_id=eos_token_id, - tie_word_embeddings=tie_word_embeddings, - dtype=dtype, - params_dtype=params_dtype, - router_dtype=router_dtype, - topk_method=topk_method, - router_bias=router_bias, - nextn_use_scmoe=nextn_use_scmoe, - **kwargs, - ) - self.vocab_size = vocab_size - self.max_position_embeddings = max_position_embeddings - self.hidden_size = hidden_size - self.num_hidden_layers = ( - num_hidden_layers if num_hidden_layers is not None else num_layers - ) - self.num_attention_heads = num_attention_heads - self.ep_size = ep_size - self.kv_lora_rank = kv_lora_rank - self.q_lora_rank = q_lora_rank - self.qk_rope_head_dim = qk_rope_head_dim - self.v_head_dim = v_head_dim - self.qk_nope_head_dim = qk_nope_head_dim - self.num_experts_per_tok = num_experts_per_tok - self.norm_topk_prob = norm_topk_prob - # for backward compatibility - if num_key_value_heads is None: - num_key_value_heads = num_attention_heads - - self.num_key_value_heads = num_key_value_heads - self.initializer_range = initializer_range - self.rms_norm_eps = rms_norm_eps - self.pretraining_tp = pretraining_tp - self.use_cache = use_cache - # Try to set `rope_scaling` if available, otherwise use `rope_parameters` - rope_scaling = kwargs.pop("rope_scaling", None) - rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"} - rope_theta = kwargs.pop("rope_theta", 1000000.0) - if "rope_theta" not in rope_parameters: - rope_parameters["rope_theta"] = rope_theta - self.rope_parameters = rope_parameters - self.attention_bias = attention_bias - self.attention_dropout = attention_dropout - self.mla_scale_q_lora = mla_scale_q_lora - self.mla_scale_kv_lora = mla_scale_kv_lora - self.zero_expert_num = zero_expert_num - self.zero_expert_type = zero_expert_type - self.routed_scaling_factor = routed_scaling_factor - self.hidden_act = "silu" - self.intermediate_size = ( - self.ffn_hidden_size - if hasattr(self, "ffn_hidden_size") - else intermediate_size - ) - if hasattr(self, "moe_intermediate_size"): - self.moe_intermediate_size = self.moe_intermediate_size - elif hasattr(self, "expert_ffn_hidden_size"): - self.moe_intermediate_size = self.expert_ffn_hidden_size - else: - self.moe_intermediate_size = self.intermediate_size - - class FlashMLP(nn.Module): """Flash MLP layer.""" @@ -242,22 +127,17 @@ class FlashMLP(nn.Module): class LongcatRouter(nn.Module): def __init__( self, - config: FlashConfig, + config: LongcatFlashConfig, zero_expert_num: int, router_params_dtype: torch.dtype, prefix: str = "", ): super().__init__() - self.n_routed_experts = ( - config.n_routed_experts - if hasattr(config, "n_routed_experts") - else config.num_experts[0] - ) - self.n_routed_experts = self.n_routed_experts + zero_expert_num + self.n_routed_experts = config.n_routed_experts + zero_expert_num self.classifier = GateLinear( config.hidden_size, self.n_routed_experts, - bias=config.router_bias, + bias=False, out_dtype=router_params_dtype, params_dtype=router_params_dtype, prefix=f"{prefix}.classifier", @@ -274,7 +154,7 @@ class LongcatRouter(nn.Module): class LongcatMoe(nn.Module): def __init__( self, - config: FlashConfig, + config: LongcatFlashConfig, num_experts: int, top_k: int, hidden_size: int, @@ -286,10 +166,7 @@ class LongcatMoe(nn.Module): ): super().__init__() self.hidden_size = hidden_size - # Gate always runs at half / full precision for now. - self.router_params_dtype = params_dtype - if config.router_dtype == "float32": - self.router_params_dtype = torch.float32 + self.router_params_dtype = torch.float32 self.router = LongcatRouter( config=config, @@ -354,7 +231,7 @@ class FlashDecoderLayer(nn.Module): def __init__( self, vllm_config: VllmConfig, - config: FlashConfig, + config: LongcatFlashConfig, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -376,9 +253,7 @@ class FlashDecoderLayer(nn.Module): qk_nope_head_dim=config.qk_nope_head_dim, qk_rope_head_dim=config.qk_rope_head_dim, v_head_dim=config.v_head_dim, - q_lora_rank=( - config.q_lora_rank if hasattr(config, "q_lora_rank") else None - ), + q_lora_rank=config.q_lora_rank, kv_lora_rank=config.kv_lora_rank, max_position_embeddings=max_position_embeddings, cache_config=cache_config, @@ -415,12 +290,8 @@ class FlashDecoderLayer(nn.Module): self.mlp = LongcatMoe( config=config, - num_experts=config.n_routed_experts - if hasattr(config, "n_routed_experts") - else config.num_experts[self.layer_idx], - top_k=config.moe_topk - if hasattr(config, "moe_topk") - else config.num_experts_per_tok, + num_experts=config.n_routed_experts, + top_k=config.moe_topk, hidden_size=config.hidden_size, intermediate_size=config.moe_intermediate_size, quant_config=quant_config, @@ -482,7 +353,7 @@ class FlashModel(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - config = FlashConfig(**vllm_config.model_config.hf_config.__dict__) + config = vllm_config.model_config.hf_config cache_config = vllm_config.cache_config quant_config = vllm_config.quant_config self.config = config @@ -499,7 +370,7 @@ class FlashModel(nn.Module): else: self.embed_tokens = PPMissingLayer() self.start_layer, self.end_layer, self.layers = make_layers( - config.num_hidden_layers, + config.num_layers, lambda prefix: FlashDecoderLayer( vllm_config, config, @@ -561,9 +432,7 @@ class FlashModel(nn.Module): ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", ckpt_up_proj_name="up_proj", - num_experts=self.config.n_routed_experts - if hasattr(self.config, "n_routed_experts") - else self.config.num_experts[0], + num_experts=self.config.n_routed_experts, ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: @@ -659,7 +528,7 @@ class FlashModel(nn.Module): ) weight_loader(param, loaded_weight) loaded_params.add(name) - for layer_id in range(self.config.num_hidden_layers): + for layer_id in range(self.config.num_layers): for i in range(2): if isinstance(self.layers[layer_id], PPMissingLayer): continue @@ -727,15 +596,10 @@ class LongcatFlashForCausalLM(nn.Module, SupportsLoRA, SupportsPP): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - config = FlashConfig(**vllm_config.model_config.hf_config.__dict__) + config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config self.config = config - config.intermediate_size = ( - config.ffn_hidden_size - if hasattr(config, "ffn_hidden_size") - else config.intermediate_size - ) self.quant_config = quant_config diff --git a/vllm/model_executor/models/longcat_flash_mtp.py b/vllm/model_executor/models/longcat_flash_mtp.py index 770f931e81..506f7a7451 100644 --- a/vllm/model_executor/models/longcat_flash_mtp.py +++ b/vllm/model_executor/models/longcat_flash_mtp.py @@ -20,7 +20,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader -from vllm.model_executor.models.longcat_flash import FlashConfig from vllm.sequence import IntermediateTensors from .deepseek_v2 import DeepseekV2DecoderLayer @@ -80,9 +79,9 @@ class LongCatMultiTokenPredictor(nn.Module): prefix: str = "", ): super().__init__() - config = FlashConfig(**vllm_config.model_config.hf_config.__dict__) - vllm_config.model_config.hf_config.intermediate_size = config.intermediate_size - self.mtp_start_layer_idx = config.num_hidden_layers * 2 + config = vllm_config.model_config.hf_config + # TODO: Use `num_hidden_layers` once Transformers 5.18.0 is the minimum + self.mtp_start_layer_idx = config.num_layers * 2 self.num_mtp_layers = 1 self.layers = torch.nn.ModuleDict( { @@ -130,7 +129,7 @@ class LongCatFlashMTP(nn.Module): # builds a dense MLP. object.__setattr__ bypasses the ngram remote # config's strict validation (it rejects setting the int field to None). object.__setattr__(vllm_config.model_config.hf_config, "n_routed_experts", None) - self.config = FlashConfig(**vllm_config.model_config.hf_config.__dict__) + self.config = vllm_config.model_config.hf_config self.quant_config = ( None if "mtp" in getattr(self.config, "disable_quant_module", []) @@ -264,7 +263,7 @@ class LongCatFlashMTP(nn.Module): weight_loader = getattr(param, "weight_loader", default_weight_loader) weight_loader(param, loaded_weight) loaded_params.add(name) - spec_layer_id = self.config.num_hidden_layers * 2 + spec_layer_id = self.model.mtp_start_layer_idx self_attn = self.model.layers[str(spec_layer_id)].mtp_block.self_attn if ( self.quant_config is not None @@ -356,5 +355,5 @@ class LongCatFlashMTP(nn.Module): self, config: PreTrainedConfig, weight_name: str ) -> int | None: if "model.mtp" in weight_name: - return config.num_hidden_layers * 2 + return self.model.mtp_start_layer_idx return None diff --git a/vllm/model_executor/models/longcat_flash_ngram.py b/vllm/model_executor/models/longcat_flash_ngram.py index 435a11ee17..aab615845d 100644 --- a/vllm/model_executor/models/longcat_flash_ngram.py +++ b/vllm/model_executor/models/longcat_flash_ngram.py @@ -15,6 +15,7 @@ from typing import Any import torch from torch import nn +from transformers import LongcatFlashConfig from vllm import _custom_ops as ops from vllm.config import VllmConfig @@ -30,15 +31,15 @@ from vllm.v1.worker.gpu.model_states.default import DefaultModelState from vllm.v1.worker.gpu.states import RequestState from .interfaces import SupportsLoRA, SupportsPP -from .longcat_flash import FlashConfig, FlashModel +from .longcat_flash import FlashModel from .utils import AutoWeightsLoader, PPMissingLayer, WeightsMapper, maybe_prefix -def uses_ngram_embedding(config: FlashConfig) -> bool: +def uses_ngram_embedding(config: LongcatFlashConfig) -> bool: return getattr(config, "ngram_vocab_size_ratio", None) is not None -def _config_dtype(config: FlashConfig) -> torch.dtype: +def _config_dtype(config: LongcatFlashConfig) -> torch.dtype: dt = getattr(config, "torch_dtype", None) or getattr(config, "dtype", None) if isinstance(dt, torch.dtype): return dt @@ -54,7 +55,7 @@ class NgramEmbedding(nn.Module): single ``bmm``. Hashing math is ported from the HF reference. """ - def __init__(self, config: FlashConfig, base_embeddings: nn.Module) -> None: + def __init__(self, config: LongcatFlashConfig, base_embeddings: nn.Module) -> None: super().__init__() self.config = config self.word_embeddings = base_embeddings @@ -170,14 +171,6 @@ class FlashNgramModel(FlashModel): """FlashModel whose input embedding is an :class:`NgramEmbedding`.""" def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: - # Each FlashDecoderLayer is a *dual* layer (2 attentions), so the number - # of decoder layers is ``num_layers``. The ngram HF config sets - # ``num_hidden_layers`` to a multiple of that (attention-module count), - # which FlashModel would otherwise build as too many (dead) layers. - hf = vllm_config.model_config.hf_config - num_layers = getattr(hf, "num_layers", None) - if num_layers is not None and hf.num_hidden_layers != num_layers: - hf.num_hidden_layers = num_layers super().__init__(vllm_config=vllm_config, prefix=prefix) self.ngram_embeddings: NgramEmbedding | None if get_pp_group().is_first_rank and uses_ngram_embedding(self.config): @@ -222,10 +215,7 @@ class LongcatFlashNgramForCausalLM(nn.Module, SupportsLoRA, SupportsPP): "V2 model runner for its n-gram embedding state; it is selected " "automatically unless VLLM_USE_V2_MODEL_RUNNER=0 is set." ) - config = FlashConfig(**vllm_config.model_config.hf_config.__dict__) - config.intermediate_size = getattr( - config, "ffn_hidden_size", config.intermediate_size - ) + config = vllm_config.model_config.hf_config self.config = config self.quant_config = vllm_config.quant_config diff --git a/vllm/models/hy_v4/nvidia/attention.py b/vllm/models/hy_v4/nvidia/attention.py index 1e9be2e3bf..48d769a856 100644 --- a/vllm/models/hy_v4/nvidia/attention.py +++ b/vllm/models/hy_v4/nvidia/attention.py @@ -47,7 +47,13 @@ from vllm.v1.attention.selector import get_attn_backend logger = init_logger(__name__) -_SPARSE_LAYER_TYPES = ("sparse_attention", "sparse", "deepseek_sparse_attention") +_SPARSE_LAYER_TYPES = ( + "sparse_attention", + "sparse", + "indexed_attention", + # TODO: Delete below once Transformers 5.18.0 is the minimum required version. + "deepseek_sparse_attention", +) _WEIGHT_LAYER_INDEX_RE = re.compile(r"(?:^|\.)layers\.(\d+)(?:\.|$)") diff --git a/vllm/models/qwen4_exp/amd/model.py b/vllm/models/qwen4_exp/amd/model.py index 16509d609c..b3a2b75053 100644 --- a/vllm/models/qwen4_exp/amd/model.py +++ b/vllm/models/qwen4_exp/amd/model.py @@ -79,6 +79,10 @@ from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm from .ple_layer import Qwen4ExpPLELayer from .qsa import Qwen4ExpQSAAttention +# Transformers v5.18 renamed `qwen_sparse_attention` to `indexed_attention` +# TODO: Delete qwen_... once Transformers 5.18.0 is the minimum required version. +_QSA_LAYER_TYPES = ("qwen_sparse_attention", "indexed_attention") + def without_modelopt_fp4( quant_config: QuantizationConfig | None, @@ -207,7 +211,7 @@ class Qwen4ExpDecoderLayer(nn.Module): prefix=f"{prefix}.linear_attn", gqa_interleaved_layout=False, ) - elif layer_type == "qwen_sparse_attention": + elif layer_type in _QSA_LAYER_TYPES: use_qsa = getattr(config, "indexer_n_heads", None) is not None if not use_qsa: self.self_attn = Qwen3NextAttention( @@ -289,7 +293,7 @@ class Qwen4ExpDecoderLayer(nn.Module): if self.layer_type == "linear_attention": attn_out = self.linear_attn(hidden_states=block_input) - elif self.layer_type == "qwen_sparse_attention": + elif self.layer_type in _QSA_LAYER_TYPES: attn_out = self.self_attn( hidden_states=block_input, positions=positions, @@ -380,7 +384,7 @@ class Qwen4ExpModel(nn.Module): self._qsa_layer_ids = frozenset( layer_idx for layer_idx, layer_type in enumerate(config.layer_types) - if layer_type == "qwen_sparse_attention" + if layer_type in _QSA_LAYER_TYPES and getattr(config, "indexer_n_heads", None) is not None ) self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size) diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index 95e920bc21..73fe2cc468 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -79,6 +79,10 @@ from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup from .ple_layer import Qwen4ExpPLELayer from .qsa import Qwen4ExpQSAAttention +# Transformers v5.18 renamed `qwen_sparse_attention` to `indexed_attention` +# TODO: Delete qwen_... once Transformers 5.18.0 is the minimum required version. +_QSA_LAYER_TYPES = ("qwen_sparse_attention", "indexed_attention") + def without_modelopt_fp4( quant_config: QuantizationConfig | None, @@ -209,7 +213,7 @@ class Qwen4ExpDecoderLayer(nn.Module): prefix=f"{prefix}.linear_attn", gqa_interleaved_layout=False, ) - elif layer_type == "qwen_sparse_attention": + elif layer_type in _QSA_LAYER_TYPES: use_qsa = getattr(config, "indexer_n_heads", None) is not None if not use_qsa: self.self_attn = Qwen3NextAttention( @@ -293,7 +297,7 @@ class Qwen4ExpDecoderLayer(nn.Module): if self.layer_type == "linear_attention": attn_out = self.linear_attn(hidden_states=block_input) - elif self.layer_type == "qwen_sparse_attention": + elif self.layer_type in _QSA_LAYER_TYPES: attn_out = self.self_attn( hidden_states=block_input, positions=positions, @@ -373,7 +377,7 @@ class Qwen4ExpModel(nn.Module): self._qsa_layer_ids = frozenset( layer_idx for layer_idx, layer_type in enumerate(config.layer_types) - if layer_type == "qwen_sparse_attention" + if layer_type in _QSA_LAYER_TYPES and getattr(config, "indexer_n_heads", None) is not None ) self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 0766f9b9ba..e49f894a85 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -147,6 +147,12 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict( _SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators", "medusa"} +# Checkpoints whose config.json has no model_type, only remote code in auto_map +_ARCHITECTURE_TO_MODEL_TYPE: dict[str, str] = { + "LongcatFlashForCausalLM": "longcat_flash", + "LongcatFlashNgramForCausalLM": "longcat_flash", +} + _PATCH_HF_VALIDATE_ROPE: set[str] = {"sarvam_mla"} # Model types whose checkpoints carry shared RoPE parameters alongside the @@ -352,6 +358,8 @@ class HFConfigParser(ConfigParserBase): if config_dict.get("speculators_config") is not None else model_type ) + if model_type is None and (architectures := config_dict.get("architectures")): + model_type = _ARCHITECTURE_TO_MODEL_TYPE.get(architectures[0]) # Allow hf_overrides to override model_type before checking _CONFIG_REGISTRY if (hf_overrides := kwargs.pop("hf_overrides", None)) is not None: if isinstance(hf_overrides, dict) and "model_type" in hf_overrides: @@ -374,6 +382,15 @@ class HFConfigParser(ConfigParserBase): if extra_layer_types := _PATCH_HF_ALLOWED_LAYER_TYPES.get(model_type): _patch_hf_transformers_allowed_layer_types(extra_layer_types) + if model_type == "vlm": + # HyperCLOVAX remote code registers this alias in a bare + # `try/except` that fails silently since transformers 5.18 + from transformers import CONFIG_MAPPING, Qwen2_5_VLVisionConfig + + CONFIG_MAPPING.register( + "qwen2_5_vl_visual", Qwen2_5_VLVisionConfig, exist_ok=True + ) + rope_parameters = config_dict.get("rope_parameters") or {} if model_type == "gemma4_text" and "full_attention" in rope_parameters: from transformers import Gemma4TextConfig @@ -387,6 +404,11 @@ class HFConfigParser(ConfigParserBase): } kwargs.setdefault("name_or_path", str(model)) config = Gemma4TextConfig.from_dict(config_dict, **kwargs) + elif model_type in _ARCHITECTURE_TO_MODEL_TYPE.values(): + from transformers import CONFIG_MAPPING + + kwargs.setdefault("name_or_path", str(model)) + config = CONFIG_MAPPING[model_type].from_dict(config_dict, **kwargs) elif model_type in _SPECULATIVE_DECODING_CONFIGS: config_class = _CONFIG_REGISTRY[model_type] config = config_class.from_pretrained( diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index 9386e9a328..9dee4af924 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -329,7 +329,6 @@ class ModelArchConfigConvertorBase: "kimi_k2", "kimi_linear", "longcat_flash", - "longcat_flash_ngram", "pangu_ultra_moe", "pangu_ultra_moe_mtp", "bailing_hybrid", @@ -714,6 +713,12 @@ class PanguUltraMoeMTPModelArchConfigConvertor(ModelArchConfigConvertorBase): return getattr(self.hf_text_config, "num_nextn_predict_layers", 0) +class LongCatFlashModelArchConfigConvertor(ModelArchConfigConvertorBase): + def get_num_hidden_layers(self) -> int: + # `num_hidden_layers` counts attention sublayers, two per decoder layer + return self.hf_text_config.num_layers + + class LongCatFlashMTPModelArchConfigConvertor(ModelArchConfigConvertorBase): def get_num_hidden_layers(self) -> int: return getattr(self.hf_text_config, "num_nextn_predict_layers", 1) @@ -825,6 +830,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = { "gemma4_unified_text": Gemma4ModelArchConfigConvertor, "glm4_moe_mtp": GLM4MoeMTPModelArchConfigConvertor, "glm_ocr_mtp": GLM4MoeMTPModelArchConfigConvertor, + "longcat_flash": LongCatFlashModelArchConfigConvertor, "longcat_flash_mtp": LongCatFlashMTPModelArchConfigConvertor, "mamba": MambaModelArchConfigConvertor, "medusa": MedusaModelArchConfigConvertor, diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index 9e9ffd7be1..4fa308bcf4 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -398,10 +398,7 @@ def init_kv_cache( # Dual-attention models (e.g. LongCat-Flash) put two Attention modules per # decoder layer, so a layer name carries two integers (layer + module index). num_attn_module = ( - 2 - if vllm_config.model_config.hf_config.model_type - in ("longcat_flash", "longcat_flash_ngram") - else 1 + 2 if vllm_config.model_config.hf_config.model_type == "longcat_flash" else 1 ) bindable_caches = { name: cache for name, cache in kv_caches.items() if name in forward_context