[CI] Bump Transformers version to 5.18.0 (#59621)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -7,7 +7,7 @@ requests >= 2.33.0
|
||||
tqdm
|
||||
blake3
|
||||
py-cpuinfo
|
||||
transformers >= 5.16.1, < 5.18.0
|
||||
transformers >= 5.16.1, < 5.19.0
|
||||
huggingface_hub >= 1.31.0 # Required for the `huggingface_hub.utils.httpx` re-export
|
||||
tokenizers >= 0.23.2 # Required for fast incremental detokenization.
|
||||
safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611
|
||||
|
||||
@@ -1228,7 +1228,7 @@ traitlets==5.16.1
|
||||
# via
|
||||
# jupyter-client
|
||||
# jupyter-core
|
||||
transformers==5.17.0
|
||||
transformers==5.18.0
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# -r requirements/test/cuda.in
|
||||
|
||||
@@ -40,7 +40,7 @@ datamodel_code_generator>=0.64.0 # required for minicpm3 test
|
||||
nltk>=3.10.3 # pin transitive rouge-score/lm-eval dep for known GHSAs
|
||||
lm-eval[api]>=0.4.12 # required for model evaluation test
|
||||
mteb[bm25s]>=2, <3 # required for mteb test
|
||||
transformers==5.17.0
|
||||
transformers==5.18.0
|
||||
schemathesis>=4.0.0 # Required for openai schema test.
|
||||
# quantization
|
||||
buildkite-test-collector==0.1.9
|
||||
|
||||
@@ -1335,7 +1335,7 @@ traitlets==5.16.1
|
||||
# via
|
||||
# jupyter-client
|
||||
# jupyter-core
|
||||
transformers==5.17.0
|
||||
transformers==5.18.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
|
||||
@@ -29,7 +29,7 @@ opencv-python-headless >= 4.13.0 # required for video test
|
||||
datamodel_code_generator>=0.64.0 # required for minicpm3 test
|
||||
lm-eval[api]>=0.4.12 # required for model evaluation test
|
||||
mteb[bm25s]>=2, <3 # required for mteb test
|
||||
transformers==5.17.0
|
||||
transformers==5.18.0
|
||||
schemathesis>=4.0.0 # Required for openai schema test.
|
||||
# quantization
|
||||
buildkite-test-collector==0.1.9
|
||||
|
||||
@@ -36,7 +36,7 @@ datamodel_code_generator>=0.64.0 # required for minicpm3 test
|
||||
nltk>=3.10.3 # pin transitive rouge-score/lm-eval dep for known GHSAs
|
||||
lm-eval[api]>=0.4.12 # required for model evaluation test
|
||||
mteb[bm25s]>=2, <3 # required for mteb test
|
||||
transformers==5.17.0
|
||||
transformers==5.18.0
|
||||
schemathesis>=4.0.0 # Required for openai schema test
|
||||
# quantization
|
||||
bitsandbytes==0.50.0
|
||||
|
||||
@@ -1282,7 +1282,7 @@ traitlets==5.16.1
|
||||
# via
|
||||
# jupyter-client
|
||||
# jupyter-core
|
||||
transformers==5.17.0
|
||||
transformers==5.18.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
|
||||
@@ -20,7 +20,7 @@ nltk>=3.10.3 # pin transitive rouge-score/lm-eval dep for known GHSAs
|
||||
modelscope<1.38
|
||||
runai-model-streamer[s3,gcs,azure]>=0.15.7
|
||||
tensorizer==2.10.1
|
||||
transformers==5.17.0
|
||||
transformers==5.18.0
|
||||
|
||||
# --- Audio Processing ---
|
||||
av
|
||||
|
||||
@@ -1134,7 +1134,7 @@ traitlets==5.16.1
|
||||
# via
|
||||
# jupyter-client
|
||||
# jupyter-core
|
||||
transformers==5.17.0
|
||||
transformers==5.18.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
|
||||
@@ -380,12 +380,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Llama4ForCausalLM": _HfExamplesInfo(
|
||||
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
||||
),
|
||||
"LongcatFlashForCausalLM": _HfExamplesInfo(
|
||||
"meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True
|
||||
),
|
||||
"LongcatFlashForCausalLM": _HfExamplesInfo("meituan-longcat/LongCat-Flash-Chat"),
|
||||
"LongcatFlashNgramForCausalLM": _HfExamplesInfo(
|
||||
"meituan-longcat/LongCat-Flash-Lite",
|
||||
trust_remote_code=True,
|
||||
# Shrink the ~62GB n-gram tables (ngram_vocab_size_ratio * vocab_size)
|
||||
# so the dummy-weight init test fits in CI memory.
|
||||
hf_overrides={"ngram_vocab_size_ratio": 1},
|
||||
@@ -1752,7 +1749,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
),
|
||||
"LongCatFlashMTPModel": _HfExamplesInfo(
|
||||
"meituan-longcat/LongCat-Flash-Chat",
|
||||
trust_remote_code=True,
|
||||
speculative_model="meituan-longcat/LongCat-Flash-Chat",
|
||||
),
|
||||
"MiMoMTPModel": _HfExamplesInfo(
|
||||
|
||||
@@ -111,6 +111,8 @@ LayerBlockType = Literal["attention", "linear_attention", "mamba"]
|
||||
_ATTENTION_LAYER_TYPES = frozenset(
|
||||
{
|
||||
"full_attention",
|
||||
"indexed_attention",
|
||||
# TODO: Delete below once Transformers 5.18.0 is the minimum required version.
|
||||
"deepseek_sparse_attention",
|
||||
"qwen_sparse_attention",
|
||||
}
|
||||
|
||||
@@ -960,7 +960,7 @@ class SpeculativeConfig:
|
||||
"architectures": [architecture],
|
||||
}
|
||||
)
|
||||
if hf_config.model_type in ("longcat_flash", "longcat_flash_ngram"):
|
||||
if hf_config.model_type == "longcat_flash":
|
||||
hf_config.model_type = "longcat_flash_mtp"
|
||||
n_predict = getattr(hf_config, "num_nextn_predict_layers", 1)
|
||||
hf_config.update(
|
||||
|
||||
@@ -50,6 +50,19 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessing
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# The checkpoint sets `do_pad: null`, which disables padding since
|
||||
# Transformers v5.18, so images of different sizes cannot be stacked.
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
hf_data,
|
||||
{**hf_kwargs, "do_pad": True},
|
||||
)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
|
||||
@@ -39,7 +39,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PreTrainedConfig
|
||||
from transformers import LongcatFlashConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
@@ -80,121 +80,6 @@ from .utils import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class FlashConfig(PreTrainedConfig):
|
||||
"""Flash model configuration."""
|
||||
|
||||
moe_intermediate_size: int
|
||||
|
||||
model_type = "longcat_flash"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=131072,
|
||||
hidden_size=4096,
|
||||
intermediate_size=8192,
|
||||
num_layers=28,
|
||||
num_hidden_layers=None,
|
||||
num_attention_heads=96,
|
||||
num_key_value_heads=128,
|
||||
ep_size=1,
|
||||
kv_lora_rank=512,
|
||||
q_lora_rank=1536,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
qk_nope_head_dim=128,
|
||||
num_experts_per_tok=None,
|
||||
norm_topk_prob=False,
|
||||
max_position_embeddings=8192,
|
||||
initializer_range=0.02,
|
||||
rms_norm_eps=1e-05,
|
||||
use_cache=True,
|
||||
pad_token_id=None,
|
||||
bos_token_id=100000,
|
||||
eos_token_id=100001,
|
||||
pretraining_tp=1,
|
||||
tie_word_embeddings=False,
|
||||
rope_parameters=None,
|
||||
attention_bias=False,
|
||||
attention_dropout=0.0,
|
||||
mla_scale_q_lora=False,
|
||||
mla_scale_kv_lora=False,
|
||||
dtype="bfloat16",
|
||||
params_dtype="bfloat16",
|
||||
router_dtype="float32",
|
||||
router_bias=False,
|
||||
topk_method=None,
|
||||
routed_scaling_factor=1.0,
|
||||
zero_expert_num=0,
|
||||
zero_expert_type=None,
|
||||
nextn_use_scmoe=False,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
dtype=dtype,
|
||||
params_dtype=params_dtype,
|
||||
router_dtype=router_dtype,
|
||||
topk_method=topk_method,
|
||||
router_bias=router_bias,
|
||||
nextn_use_scmoe=nextn_use_scmoe,
|
||||
**kwargs,
|
||||
)
|
||||
self.vocab_size = vocab_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.num_hidden_layers = (
|
||||
num_hidden_layers if num_hidden_layers is not None else num_layers
|
||||
)
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.ep_size = ep_size
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
self.norm_topk_prob = norm_topk_prob
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.initializer_range = initializer_range
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.pretraining_tp = pretraining_tp
|
||||
self.use_cache = use_cache
|
||||
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
|
||||
rope_scaling = kwargs.pop("rope_scaling", None)
|
||||
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
|
||||
rope_theta = kwargs.pop("rope_theta", 1000000.0)
|
||||
if "rope_theta" not in rope_parameters:
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
self.rope_parameters = rope_parameters
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
self.mla_scale_q_lora = mla_scale_q_lora
|
||||
self.mla_scale_kv_lora = mla_scale_kv_lora
|
||||
self.zero_expert_num = zero_expert_num
|
||||
self.zero_expert_type = zero_expert_type
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.hidden_act = "silu"
|
||||
self.intermediate_size = (
|
||||
self.ffn_hidden_size
|
||||
if hasattr(self, "ffn_hidden_size")
|
||||
else intermediate_size
|
||||
)
|
||||
if hasattr(self, "moe_intermediate_size"):
|
||||
self.moe_intermediate_size = self.moe_intermediate_size
|
||||
elif hasattr(self, "expert_ffn_hidden_size"):
|
||||
self.moe_intermediate_size = self.expert_ffn_hidden_size
|
||||
else:
|
||||
self.moe_intermediate_size = self.intermediate_size
|
||||
|
||||
|
||||
class FlashMLP(nn.Module):
|
||||
"""Flash MLP layer."""
|
||||
|
||||
@@ -242,22 +127,17 @@ class FlashMLP(nn.Module):
|
||||
class LongcatRouter(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: FlashConfig,
|
||||
config: LongcatFlashConfig,
|
||||
zero_expert_num: int,
|
||||
router_params_dtype: torch.dtype,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.n_routed_experts = (
|
||||
config.n_routed_experts
|
||||
if hasattr(config, "n_routed_experts")
|
||||
else config.num_experts[0]
|
||||
)
|
||||
self.n_routed_experts = self.n_routed_experts + zero_expert_num
|
||||
self.n_routed_experts = config.n_routed_experts + zero_expert_num
|
||||
self.classifier = GateLinear(
|
||||
config.hidden_size,
|
||||
self.n_routed_experts,
|
||||
bias=config.router_bias,
|
||||
bias=False,
|
||||
out_dtype=router_params_dtype,
|
||||
params_dtype=router_params_dtype,
|
||||
prefix=f"{prefix}.classifier",
|
||||
@@ -274,7 +154,7 @@ class LongcatRouter(nn.Module):
|
||||
class LongcatMoe(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: FlashConfig,
|
||||
config: LongcatFlashConfig,
|
||||
num_experts: int,
|
||||
top_k: int,
|
||||
hidden_size: int,
|
||||
@@ -286,10 +166,7 @@ class LongcatMoe(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
# Gate always runs at half / full precision for now.
|
||||
self.router_params_dtype = params_dtype
|
||||
if config.router_dtype == "float32":
|
||||
self.router_params_dtype = torch.float32
|
||||
self.router_params_dtype = torch.float32
|
||||
|
||||
self.router = LongcatRouter(
|
||||
config=config,
|
||||
@@ -354,7 +231,7 @@ class FlashDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: FlashConfig,
|
||||
config: LongcatFlashConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
@@ -376,9 +253,7 @@ class FlashDecoderLayer(nn.Module):
|
||||
qk_nope_head_dim=config.qk_nope_head_dim,
|
||||
qk_rope_head_dim=config.qk_rope_head_dim,
|
||||
v_head_dim=config.v_head_dim,
|
||||
q_lora_rank=(
|
||||
config.q_lora_rank if hasattr(config, "q_lora_rank") else None
|
||||
),
|
||||
q_lora_rank=config.q_lora_rank,
|
||||
kv_lora_rank=config.kv_lora_rank,
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
cache_config=cache_config,
|
||||
@@ -415,12 +290,8 @@ class FlashDecoderLayer(nn.Module):
|
||||
|
||||
self.mlp = LongcatMoe(
|
||||
config=config,
|
||||
num_experts=config.n_routed_experts
|
||||
if hasattr(config, "n_routed_experts")
|
||||
else config.num_experts[self.layer_idx],
|
||||
top_k=config.moe_topk
|
||||
if hasattr(config, "moe_topk")
|
||||
else config.num_experts_per_tok,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.moe_topk,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
quant_config=quant_config,
|
||||
@@ -482,7 +353,7 @@ class FlashModel(nn.Module):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
|
||||
config = vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
@@ -499,7 +370,7 @@ class FlashModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
config.num_layers,
|
||||
lambda prefix: FlashDecoderLayer(
|
||||
vllm_config,
|
||||
config,
|
||||
@@ -561,9 +432,7 @@ class FlashModel(nn.Module):
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.n_routed_experts
|
||||
if hasattr(self.config, "n_routed_experts")
|
||||
else self.config.num_experts[0],
|
||||
num_experts=self.config.n_routed_experts,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
@@ -659,7 +528,7 @@ class FlashModel(nn.Module):
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
for layer_id in range(self.config.num_hidden_layers):
|
||||
for layer_id in range(self.config.num_layers):
|
||||
for i in range(2):
|
||||
if isinstance(self.layers[layer_id], PPMissingLayer):
|
||||
continue
|
||||
@@ -727,15 +596,10 @@ class LongcatFlashForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
config.intermediate_size = (
|
||||
config.ffn_hidden_size
|
||||
if hasattr(config, "ffn_hidden_size")
|
||||
else config.intermediate_size
|
||||
)
|
||||
|
||||
self.quant_config = quant_config
|
||||
|
||||
|
||||
@@ -20,7 +20,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.longcat_flash import FlashConfig
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .deepseek_v2 import DeepseekV2DecoderLayer
|
||||
@@ -80,9 +79,9 @@ class LongCatMultiTokenPredictor(nn.Module):
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
|
||||
vllm_config.model_config.hf_config.intermediate_size = config.intermediate_size
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers * 2
|
||||
config = vllm_config.model_config.hf_config
|
||||
# TODO: Use `num_hidden_layers` once Transformers 5.18.0 is the minimum
|
||||
self.mtp_start_layer_idx = config.num_layers * 2
|
||||
self.num_mtp_layers = 1
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
{
|
||||
@@ -130,7 +129,7 @@ class LongCatFlashMTP(nn.Module):
|
||||
# builds a dense MLP. object.__setattr__ bypasses the ngram remote
|
||||
# config's strict validation (it rejects setting the int field to None).
|
||||
object.__setattr__(vllm_config.model_config.hf_config, "n_routed_experts", None)
|
||||
self.config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.quant_config = (
|
||||
None
|
||||
if "mtp" in getattr(self.config, "disable_quant_module", [])
|
||||
@@ -264,7 +263,7 @@ class LongCatFlashMTP(nn.Module):
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
spec_layer_id = self.config.num_hidden_layers * 2
|
||||
spec_layer_id = self.model.mtp_start_layer_idx
|
||||
self_attn = self.model.layers[str(spec_layer_id)].mtp_block.self_attn
|
||||
if (
|
||||
self.quant_config is not None
|
||||
@@ -356,5 +355,5 @@ class LongCatFlashMTP(nn.Module):
|
||||
self, config: PreTrainedConfig, weight_name: str
|
||||
) -> int | None:
|
||||
if "model.mtp" in weight_name:
|
||||
return config.num_hidden_layers * 2
|
||||
return self.model.mtp_start_layer_idx
|
||||
return None
|
||||
|
||||
@@ -15,6 +15,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import LongcatFlashConfig
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import VllmConfig
|
||||
@@ -30,15 +31,15 @@ from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .longcat_flash import FlashConfig, FlashModel
|
||||
from .longcat_flash import FlashModel
|
||||
from .utils import AutoWeightsLoader, PPMissingLayer, WeightsMapper, maybe_prefix
|
||||
|
||||
|
||||
def uses_ngram_embedding(config: FlashConfig) -> bool:
|
||||
def uses_ngram_embedding(config: LongcatFlashConfig) -> bool:
|
||||
return getattr(config, "ngram_vocab_size_ratio", None) is not None
|
||||
|
||||
|
||||
def _config_dtype(config: FlashConfig) -> torch.dtype:
|
||||
def _config_dtype(config: LongcatFlashConfig) -> torch.dtype:
|
||||
dt = getattr(config, "torch_dtype", None) or getattr(config, "dtype", None)
|
||||
if isinstance(dt, torch.dtype):
|
||||
return dt
|
||||
@@ -54,7 +55,7 @@ class NgramEmbedding(nn.Module):
|
||||
single ``bmm``. Hashing math is ported from the HF reference.
|
||||
"""
|
||||
|
||||
def __init__(self, config: FlashConfig, base_embeddings: nn.Module) -> None:
|
||||
def __init__(self, config: LongcatFlashConfig, base_embeddings: nn.Module) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.word_embeddings = base_embeddings
|
||||
@@ -170,14 +171,6 @@ class FlashNgramModel(FlashModel):
|
||||
"""FlashModel whose input embedding is an :class:`NgramEmbedding`."""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
# Each FlashDecoderLayer is a *dual* layer (2 attentions), so the number
|
||||
# of decoder layers is ``num_layers``. The ngram HF config sets
|
||||
# ``num_hidden_layers`` to a multiple of that (attention-module count),
|
||||
# which FlashModel would otherwise build as too many (dead) layers.
|
||||
hf = vllm_config.model_config.hf_config
|
||||
num_layers = getattr(hf, "num_layers", None)
|
||||
if num_layers is not None and hf.num_hidden_layers != num_layers:
|
||||
hf.num_hidden_layers = num_layers
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
self.ngram_embeddings: NgramEmbedding | None
|
||||
if get_pp_group().is_first_rank and uses_ngram_embedding(self.config):
|
||||
@@ -222,10 +215,7 @@ class LongcatFlashNgramForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
"V2 model runner for its n-gram embedding state; it is selected "
|
||||
"automatically unless VLLM_USE_V2_MODEL_RUNNER=0 is set."
|
||||
)
|
||||
config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
|
||||
config.intermediate_size = getattr(
|
||||
config, "ffn_hidden_size", config.intermediate_size
|
||||
)
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.config = config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
|
||||
@@ -47,7 +47,13 @@ from vllm.v1.attention.selector import get_attn_backend
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_SPARSE_LAYER_TYPES = ("sparse_attention", "sparse", "deepseek_sparse_attention")
|
||||
_SPARSE_LAYER_TYPES = (
|
||||
"sparse_attention",
|
||||
"sparse",
|
||||
"indexed_attention",
|
||||
# TODO: Delete below once Transformers 5.18.0 is the minimum required version.
|
||||
"deepseek_sparse_attention",
|
||||
)
|
||||
_WEIGHT_LAYER_INDEX_RE = re.compile(r"(?:^|\.)layers\.(\d+)(?:\.|$)")
|
||||
|
||||
|
||||
|
||||
@@ -79,6 +79,10 @@ from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
from .ple_layer import Qwen4ExpPLELayer
|
||||
from .qsa import Qwen4ExpQSAAttention
|
||||
|
||||
# Transformers v5.18 renamed `qwen_sparse_attention` to `indexed_attention`
|
||||
# TODO: Delete qwen_... once Transformers 5.18.0 is the minimum required version.
|
||||
_QSA_LAYER_TYPES = ("qwen_sparse_attention", "indexed_attention")
|
||||
|
||||
|
||||
def without_modelopt_fp4(
|
||||
quant_config: QuantizationConfig | None,
|
||||
@@ -207,7 +211,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
gqa_interleaved_layout=False,
|
||||
)
|
||||
elif layer_type == "qwen_sparse_attention":
|
||||
elif layer_type in _QSA_LAYER_TYPES:
|
||||
use_qsa = getattr(config, "indexer_n_heads", None) is not None
|
||||
if not use_qsa:
|
||||
self.self_attn = Qwen3NextAttention(
|
||||
@@ -289,7 +293,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
|
||||
if self.layer_type == "linear_attention":
|
||||
attn_out = self.linear_attn(hidden_states=block_input)
|
||||
elif self.layer_type == "qwen_sparse_attention":
|
||||
elif self.layer_type in _QSA_LAYER_TYPES:
|
||||
attn_out = self.self_attn(
|
||||
hidden_states=block_input,
|
||||
positions=positions,
|
||||
@@ -380,7 +384,7 @@ class Qwen4ExpModel(nn.Module):
|
||||
self._qsa_layer_ids = frozenset(
|
||||
layer_idx
|
||||
for layer_idx, layer_type in enumerate(config.layer_types)
|
||||
if layer_type == "qwen_sparse_attention"
|
||||
if layer_type in _QSA_LAYER_TYPES
|
||||
and getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
|
||||
|
||||
@@ -79,6 +79,10 @@ from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup
|
||||
from .ple_layer import Qwen4ExpPLELayer
|
||||
from .qsa import Qwen4ExpQSAAttention
|
||||
|
||||
# Transformers v5.18 renamed `qwen_sparse_attention` to `indexed_attention`
|
||||
# TODO: Delete qwen_... once Transformers 5.18.0 is the minimum required version.
|
||||
_QSA_LAYER_TYPES = ("qwen_sparse_attention", "indexed_attention")
|
||||
|
||||
|
||||
def without_modelopt_fp4(
|
||||
quant_config: QuantizationConfig | None,
|
||||
@@ -209,7 +213,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
gqa_interleaved_layout=False,
|
||||
)
|
||||
elif layer_type == "qwen_sparse_attention":
|
||||
elif layer_type in _QSA_LAYER_TYPES:
|
||||
use_qsa = getattr(config, "indexer_n_heads", None) is not None
|
||||
if not use_qsa:
|
||||
self.self_attn = Qwen3NextAttention(
|
||||
@@ -293,7 +297,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
|
||||
if self.layer_type == "linear_attention":
|
||||
attn_out = self.linear_attn(hidden_states=block_input)
|
||||
elif self.layer_type == "qwen_sparse_attention":
|
||||
elif self.layer_type in _QSA_LAYER_TYPES:
|
||||
attn_out = self.self_attn(
|
||||
hidden_states=block_input,
|
||||
positions=positions,
|
||||
@@ -373,7 +377,7 @@ class Qwen4ExpModel(nn.Module):
|
||||
self._qsa_layer_ids = frozenset(
|
||||
layer_idx
|
||||
for layer_idx, layer_type in enumerate(config.layer_types)
|
||||
if layer_type == "qwen_sparse_attention"
|
||||
if layer_type in _QSA_LAYER_TYPES
|
||||
and getattr(config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size)
|
||||
|
||||
@@ -147,6 +147,12 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict(
|
||||
|
||||
_SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators", "medusa"}
|
||||
|
||||
# Checkpoints whose config.json has no model_type, only remote code in auto_map
|
||||
_ARCHITECTURE_TO_MODEL_TYPE: dict[str, str] = {
|
||||
"LongcatFlashForCausalLM": "longcat_flash",
|
||||
"LongcatFlashNgramForCausalLM": "longcat_flash",
|
||||
}
|
||||
|
||||
_PATCH_HF_VALIDATE_ROPE: set[str] = {"sarvam_mla"}
|
||||
|
||||
# Model types whose checkpoints carry shared RoPE parameters alongside the
|
||||
@@ -352,6 +358,8 @@ class HFConfigParser(ConfigParserBase):
|
||||
if config_dict.get("speculators_config") is not None
|
||||
else model_type
|
||||
)
|
||||
if model_type is None and (architectures := config_dict.get("architectures")):
|
||||
model_type = _ARCHITECTURE_TO_MODEL_TYPE.get(architectures[0])
|
||||
# Allow hf_overrides to override model_type before checking _CONFIG_REGISTRY
|
||||
if (hf_overrides := kwargs.pop("hf_overrides", None)) is not None:
|
||||
if isinstance(hf_overrides, dict) and "model_type" in hf_overrides:
|
||||
@@ -374,6 +382,15 @@ class HFConfigParser(ConfigParserBase):
|
||||
if extra_layer_types := _PATCH_HF_ALLOWED_LAYER_TYPES.get(model_type):
|
||||
_patch_hf_transformers_allowed_layer_types(extra_layer_types)
|
||||
|
||||
if model_type == "vlm":
|
||||
# HyperCLOVAX remote code registers this alias in a bare
|
||||
# `try/except` that fails silently since transformers 5.18
|
||||
from transformers import CONFIG_MAPPING, Qwen2_5_VLVisionConfig
|
||||
|
||||
CONFIG_MAPPING.register(
|
||||
"qwen2_5_vl_visual", Qwen2_5_VLVisionConfig, exist_ok=True
|
||||
)
|
||||
|
||||
rope_parameters = config_dict.get("rope_parameters") or {}
|
||||
if model_type == "gemma4_text" and "full_attention" in rope_parameters:
|
||||
from transformers import Gemma4TextConfig
|
||||
@@ -387,6 +404,11 @@ class HFConfigParser(ConfigParserBase):
|
||||
}
|
||||
kwargs.setdefault("name_or_path", str(model))
|
||||
config = Gemma4TextConfig.from_dict(config_dict, **kwargs)
|
||||
elif model_type in _ARCHITECTURE_TO_MODEL_TYPE.values():
|
||||
from transformers import CONFIG_MAPPING
|
||||
|
||||
kwargs.setdefault("name_or_path", str(model))
|
||||
config = CONFIG_MAPPING[model_type].from_dict(config_dict, **kwargs)
|
||||
elif model_type in _SPECULATIVE_DECODING_CONFIGS:
|
||||
config_class = _CONFIG_REGISTRY[model_type]
|
||||
config = config_class.from_pretrained(
|
||||
|
||||
@@ -329,7 +329,6 @@ class ModelArchConfigConvertorBase:
|
||||
"kimi_k2",
|
||||
"kimi_linear",
|
||||
"longcat_flash",
|
||||
"longcat_flash_ngram",
|
||||
"pangu_ultra_moe",
|
||||
"pangu_ultra_moe_mtp",
|
||||
"bailing_hybrid",
|
||||
@@ -714,6 +713,12 @@ class PanguUltraMoeMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
return getattr(self.hf_text_config, "num_nextn_predict_layers", 0)
|
||||
|
||||
|
||||
class LongCatFlashModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
def get_num_hidden_layers(self) -> int:
|
||||
# `num_hidden_layers` counts attention sublayers, two per decoder layer
|
||||
return self.hf_text_config.num_layers
|
||||
|
||||
|
||||
class LongCatFlashMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
def get_num_hidden_layers(self) -> int:
|
||||
return getattr(self.hf_text_config, "num_nextn_predict_layers", 1)
|
||||
@@ -825,6 +830,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
|
||||
"gemma4_unified_text": Gemma4ModelArchConfigConvertor,
|
||||
"glm4_moe_mtp": GLM4MoeMTPModelArchConfigConvertor,
|
||||
"glm_ocr_mtp": GLM4MoeMTPModelArchConfigConvertor,
|
||||
"longcat_flash": LongCatFlashModelArchConfigConvertor,
|
||||
"longcat_flash_mtp": LongCatFlashMTPModelArchConfigConvertor,
|
||||
"mamba": MambaModelArchConfigConvertor,
|
||||
"medusa": MedusaModelArchConfigConvertor,
|
||||
|
||||
@@ -398,10 +398,7 @@ def init_kv_cache(
|
||||
# Dual-attention models (e.g. LongCat-Flash) put two Attention modules per
|
||||
# decoder layer, so a layer name carries two integers (layer + module index).
|
||||
num_attn_module = (
|
||||
2
|
||||
if vllm_config.model_config.hf_config.model_type
|
||||
in ("longcat_flash", "longcat_flash_ngram")
|
||||
else 1
|
||||
2 if vllm_config.model_config.hf_config.model_type == "longcat_flash" else 1
|
||||
)
|
||||
bindable_caches = {
|
||||
name: cache for name, cache in kv_caches.items() if name in forward_context
|
||||
|
||||
Reference in New Issue
Block a user