Merge remote-tracking branch 'upstream/main' into patches

This commit is contained in:
2026-10-05 08:52:31 +07:00
85 changed files with 3146 additions and 769 deletions
@@ -918,7 +918,7 @@ def mean_batch_invariant(input, dim, keepdim=False, dtype: torch.dtype | None =
for d in sorted_dims:
result = result.squeeze(d)
return result
return result if dtype is not None else result.to(input.dtype)
@triton.jit
+13
View File
@@ -50,6 +50,19 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessing
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _call_hf_processor(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
# The checkpoint sets `do_pad: null`, which disables padding since
# Transformers v5.18, so images of different sizes cannot be stacked.
return self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_kwargs),
hf_data,
{**hf_kwargs, "do_pad": True},
)
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
+15 -151
View File
@@ -39,7 +39,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import PreTrainedConfig
from transformers import LongcatFlashConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
@@ -80,121 +80,6 @@ from .utils import (
logger = init_logger(__name__)
class FlashConfig(PreTrainedConfig):
"""Flash model configuration."""
moe_intermediate_size: int
model_type = "longcat_flash"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
vocab_size=131072,
hidden_size=4096,
intermediate_size=8192,
num_layers=28,
num_hidden_layers=None,
num_attention_heads=96,
num_key_value_heads=128,
ep_size=1,
kv_lora_rank=512,
q_lora_rank=1536,
qk_rope_head_dim=64,
v_head_dim=128,
qk_nope_head_dim=128,
num_experts_per_tok=None,
norm_topk_prob=False,
max_position_embeddings=8192,
initializer_range=0.02,
rms_norm_eps=1e-05,
use_cache=True,
pad_token_id=None,
bos_token_id=100000,
eos_token_id=100001,
pretraining_tp=1,
tie_word_embeddings=False,
rope_parameters=None,
attention_bias=False,
attention_dropout=0.0,
mla_scale_q_lora=False,
mla_scale_kv_lora=False,
dtype="bfloat16",
params_dtype="bfloat16",
router_dtype="float32",
router_bias=False,
topk_method=None,
routed_scaling_factor=1.0,
zero_expert_num=0,
zero_expert_type=None,
nextn_use_scmoe=False,
**kwargs,
):
super().__init__(
pad_token_id=pad_token_id,
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
dtype=dtype,
params_dtype=params_dtype,
router_dtype=router_dtype,
topk_method=topk_method,
router_bias=router_bias,
nextn_use_scmoe=nextn_use_scmoe,
**kwargs,
)
self.vocab_size = vocab_size
self.max_position_embeddings = max_position_embeddings
self.hidden_size = hidden_size
self.num_hidden_layers = (
num_hidden_layers if num_hidden_layers is not None else num_layers
)
self.num_attention_heads = num_attention_heads
self.ep_size = ep_size
self.kv_lora_rank = kv_lora_rank
self.q_lora_rank = q_lora_rank
self.qk_rope_head_dim = qk_rope_head_dim
self.v_head_dim = v_head_dim
self.qk_nope_head_dim = qk_nope_head_dim
self.num_experts_per_tok = num_experts_per_tok
self.norm_topk_prob = norm_topk_prob
# for backward compatibility
if num_key_value_heads is None:
num_key_value_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.pretraining_tp = pretraining_tp
self.use_cache = use_cache
# Try to set `rope_scaling` if available, otherwise use `rope_parameters`
rope_scaling = kwargs.pop("rope_scaling", None)
rope_parameters = rope_scaling or rope_parameters or {"rope_type": "default"}
rope_theta = kwargs.pop("rope_theta", 1000000.0)
if "rope_theta" not in rope_parameters:
rope_parameters["rope_theta"] = rope_theta
self.rope_parameters = rope_parameters
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
self.mla_scale_q_lora = mla_scale_q_lora
self.mla_scale_kv_lora = mla_scale_kv_lora
self.zero_expert_num = zero_expert_num
self.zero_expert_type = zero_expert_type
self.routed_scaling_factor = routed_scaling_factor
self.hidden_act = "silu"
self.intermediate_size = (
self.ffn_hidden_size
if hasattr(self, "ffn_hidden_size")
else intermediate_size
)
if hasattr(self, "moe_intermediate_size"):
self.moe_intermediate_size = self.moe_intermediate_size
elif hasattr(self, "expert_ffn_hidden_size"):
self.moe_intermediate_size = self.expert_ffn_hidden_size
else:
self.moe_intermediate_size = self.intermediate_size
class FlashMLP(nn.Module):
"""Flash MLP layer."""
@@ -242,22 +127,17 @@ class FlashMLP(nn.Module):
class LongcatRouter(nn.Module):
def __init__(
self,
config: FlashConfig,
config: LongcatFlashConfig,
zero_expert_num: int,
router_params_dtype: torch.dtype,
prefix: str = "",
):
super().__init__()
self.n_routed_experts = (
config.n_routed_experts
if hasattr(config, "n_routed_experts")
else config.num_experts[0]
)
self.n_routed_experts = self.n_routed_experts + zero_expert_num
self.n_routed_experts = config.n_routed_experts + zero_expert_num
self.classifier = GateLinear(
config.hidden_size,
self.n_routed_experts,
bias=config.router_bias,
bias=False,
out_dtype=router_params_dtype,
params_dtype=router_params_dtype,
prefix=f"{prefix}.classifier",
@@ -274,7 +154,7 @@ class LongcatRouter(nn.Module):
class LongcatMoe(nn.Module):
def __init__(
self,
config: FlashConfig,
config: LongcatFlashConfig,
num_experts: int,
top_k: int,
hidden_size: int,
@@ -286,10 +166,7 @@ class LongcatMoe(nn.Module):
):
super().__init__()
self.hidden_size = hidden_size
# Gate always runs at half / full precision for now.
self.router_params_dtype = params_dtype
if config.router_dtype == "float32":
self.router_params_dtype = torch.float32
self.router_params_dtype = torch.float32
self.router = LongcatRouter(
config=config,
@@ -354,7 +231,7 @@ class FlashDecoderLayer(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
config: FlashConfig,
config: LongcatFlashConfig,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
@@ -376,9 +253,7 @@ class FlashDecoderLayer(nn.Module):
qk_nope_head_dim=config.qk_nope_head_dim,
qk_rope_head_dim=config.qk_rope_head_dim,
v_head_dim=config.v_head_dim,
q_lora_rank=(
config.q_lora_rank if hasattr(config, "q_lora_rank") else None
),
q_lora_rank=config.q_lora_rank,
kv_lora_rank=config.kv_lora_rank,
max_position_embeddings=max_position_embeddings,
cache_config=cache_config,
@@ -415,12 +290,8 @@ class FlashDecoderLayer(nn.Module):
self.mlp = LongcatMoe(
config=config,
num_experts=config.n_routed_experts
if hasattr(config, "n_routed_experts")
else config.num_experts[self.layer_idx],
top_k=config.moe_topk
if hasattr(config, "moe_topk")
else config.num_experts_per_tok,
num_experts=config.n_routed_experts,
top_k=config.moe_topk,
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
quant_config=quant_config,
@@ -482,7 +353,7 @@ class FlashModel(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
config = vllm_config.model_config.hf_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.config = config
@@ -499,7 +370,7 @@ class FlashModel(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
config.num_layers,
lambda prefix: FlashDecoderLayer(
vllm_config,
config,
@@ -561,9 +432,7 @@ class FlashModel(nn.Module):
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts
if hasattr(self.config, "n_routed_experts")
else self.config.num_experts[0],
num_experts=self.config.n_routed_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -659,7 +528,7 @@ class FlashModel(nn.Module):
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
for layer_id in range(self.config.num_hidden_layers):
for layer_id in range(self.config.num_layers):
for i in range(2):
if isinstance(self.layers[layer_id], PPMissingLayer):
continue
@@ -727,15 +596,10 @@ class LongcatFlashForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
config.intermediate_size = (
config.ffn_hidden_size
if hasattr(config, "ffn_hidden_size")
else config.intermediate_size
)
self.quant_config = quant_config
@@ -20,7 +20,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.longcat_flash import FlashConfig
from vllm.sequence import IntermediateTensors
from .deepseek_v2 import DeepseekV2DecoderLayer
@@ -80,9 +79,9 @@ class LongCatMultiTokenPredictor(nn.Module):
prefix: str = "",
):
super().__init__()
config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
vllm_config.model_config.hf_config.intermediate_size = config.intermediate_size
self.mtp_start_layer_idx = config.num_hidden_layers * 2
config = vllm_config.model_config.hf_config
# TODO: Use `num_hidden_layers` once Transformers 5.18.0 is the minimum
self.mtp_start_layer_idx = config.num_layers * 2
self.num_mtp_layers = 1
self.layers = torch.nn.ModuleDict(
{
@@ -130,7 +129,7 @@ class LongCatFlashMTP(nn.Module):
# builds a dense MLP. object.__setattr__ bypasses the ngram remote
# config's strict validation (it rejects setting the int field to None).
object.__setattr__(vllm_config.model_config.hf_config, "n_routed_experts", None)
self.config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
self.config = vllm_config.model_config.hf_config
self.quant_config = (
None
if "mtp" in getattr(self.config, "disable_quant_module", [])
@@ -264,7 +263,7 @@ class LongCatFlashMTP(nn.Module):
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
spec_layer_id = self.config.num_hidden_layers * 2
spec_layer_id = self.model.mtp_start_layer_idx
self_attn = self.model.layers[str(spec_layer_id)].mtp_block.self_attn
if (
self.quant_config is not None
@@ -356,5 +355,5 @@ class LongCatFlashMTP(nn.Module):
self, config: PreTrainedConfig, weight_name: str
) -> int | None:
if "model.mtp" in weight_name:
return config.num_hidden_layers * 2
return self.model.mtp_start_layer_idx
return None
@@ -15,6 +15,7 @@ from typing import Any
import torch
from torch import nn
from transformers import LongcatFlashConfig
from vllm import _custom_ops as ops
from vllm.config import VllmConfig
@@ -30,15 +31,15 @@ from vllm.v1.worker.gpu.model_states.default import DefaultModelState
from vllm.v1.worker.gpu.states import RequestState
from .interfaces import SupportsLoRA, SupportsPP
from .longcat_flash import FlashConfig, FlashModel
from .longcat_flash import FlashModel
from .utils import AutoWeightsLoader, PPMissingLayer, WeightsMapper, maybe_prefix
def uses_ngram_embedding(config: FlashConfig) -> bool:
def uses_ngram_embedding(config: LongcatFlashConfig) -> bool:
return getattr(config, "ngram_vocab_size_ratio", None) is not None
def _config_dtype(config: FlashConfig) -> torch.dtype:
def _config_dtype(config: LongcatFlashConfig) -> torch.dtype:
dt = getattr(config, "torch_dtype", None) or getattr(config, "dtype", None)
if isinstance(dt, torch.dtype):
return dt
@@ -54,7 +55,7 @@ class NgramEmbedding(nn.Module):
single ``bmm``. Hashing math is ported from the HF reference.
"""
def __init__(self, config: FlashConfig, base_embeddings: nn.Module) -> None:
def __init__(self, config: LongcatFlashConfig, base_embeddings: nn.Module) -> None:
super().__init__()
self.config = config
self.word_embeddings = base_embeddings
@@ -170,14 +171,6 @@ class FlashNgramModel(FlashModel):
"""FlashModel whose input embedding is an :class:`NgramEmbedding`."""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
# Each FlashDecoderLayer is a *dual* layer (2 attentions), so the number
# of decoder layers is ``num_layers``. The ngram HF config sets
# ``num_hidden_layers`` to a multiple of that (attention-module count),
# which FlashModel would otherwise build as too many (dead) layers.
hf = vllm_config.model_config.hf_config
num_layers = getattr(hf, "num_layers", None)
if num_layers is not None and hf.num_hidden_layers != num_layers:
hf.num_hidden_layers = num_layers
super().__init__(vllm_config=vllm_config, prefix=prefix)
self.ngram_embeddings: NgramEmbedding | None
if get_pp_group().is_first_rank and uses_ngram_embedding(self.config):
@@ -222,10 +215,7 @@ class LongcatFlashNgramForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
"V2 model runner for its n-gram embedding state; it is selected "
"automatically unless VLLM_USE_V2_MODEL_RUNNER=0 is set."
)
config = FlashConfig(**vllm_config.model_config.hf_config.__dict__)
config.intermediate_size = getattr(
config, "ffn_hidden_size", config.intermediate_size
)
config = vllm_config.model_config.hf_config
self.config = config
self.quant_config = vllm_config.quant_config