[Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
866f02db8a
commit
e3cae8d2ac
@@ -4,6 +4,7 @@
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from transformers import Glm5NextTextConfig
|
||||
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
@@ -32,7 +33,6 @@ from vllm.model_executor.utils import maybe_disable_graph_partition
|
||||
from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8
|
||||
from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
|
||||
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec
|
||||
@@ -221,7 +221,7 @@ class Indexer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: Glm5NextConfig,
|
||||
config: Glm5NextTextConfig,
|
||||
hidden_size: int,
|
||||
q_lora_rank: int,
|
||||
quant_config: QuantizationConfig | None,
|
||||
@@ -236,7 +236,7 @@ class Indexer(nn.Module):
|
||||
# self.indexer_cfg = config.attn_module_list_cfg[0]["attn_index"]
|
||||
# Indexer is only constructed for v32 configs, where these sparse-indexer
|
||||
# fields are guaranteed populated; narrow away the `int | None` declared
|
||||
# on Glm5NextConfig for the optional-indexer case.
|
||||
# on Glm5NextTextConfig for the optional-indexer case.
|
||||
assert config.index_topk is not None
|
||||
assert config.index_n_heads is not None
|
||||
assert config.index_head_dim is not None
|
||||
@@ -419,7 +419,7 @@ class Glm5NextMLAAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: Glm5NextConfig,
|
||||
config: Glm5NextTextConfig,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
qk_nope_head_dim: int,
|
||||
@@ -506,28 +506,32 @@ class Glm5NextMLAAttention(nn.Module):
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
# `Glm5NextTextConfig` is NoPE-only and so declares no `rope_parameters`;
|
||||
# a checkpoint may still ship one.
|
||||
rope_parameters = getattr(config, "rope_parameters", None)
|
||||
|
||||
if not skip_rope:
|
||||
assert config.rope_parameters is not None
|
||||
if config.rope_parameters["rope_type"] != "default":
|
||||
config.rope_parameters["rope_type"] = (
|
||||
assert rope_parameters is not None
|
||||
if rope_parameters["rope_type"] != "default":
|
||||
rope_parameters["rope_type"] = (
|
||||
"deepseek_llama_scaling"
|
||||
if config.rope_parameters.get("attention_factor") == 1.0
|
||||
if rope_parameters.get("attention_factor") == 1.0
|
||||
else "deepseek_yarn"
|
||||
)
|
||||
|
||||
self.rotary_emb: RotaryEmbedding | None = get_rope(
|
||||
qk_rope_head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
rope_parameters=rope_parameters,
|
||||
is_neox_style=False,
|
||||
)
|
||||
|
||||
if (
|
||||
config.rope_parameters["rope_type"] != "default"
|
||||
and config.rope_parameters["rope_type"] == "deepseek_yarn"
|
||||
rope_parameters["rope_type"] != "default"
|
||||
and rope_parameters["rope_type"] == "deepseek_yarn"
|
||||
):
|
||||
mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False)
|
||||
scaling_factor = config.rope_parameters["factor"]
|
||||
mscale_all_dim = rope_parameters.get("mscale_all_dim", False)
|
||||
scaling_factor = rope_parameters["factor"]
|
||||
mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
|
||||
self.scaling = self.scaling * mscale * mscale
|
||||
else:
|
||||
@@ -539,7 +543,7 @@ class Glm5NextMLAAttention(nn.Module):
|
||||
self.indexer_rope_emb: RotaryEmbedding | None = get_rope(
|
||||
qk_rope_head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
rope_parameters=rope_parameters,
|
||||
is_neox_style=not config.indexer_rope_interleave,
|
||||
)
|
||||
# The sparse indexer projects from the MLA q-lora rank, which is
|
||||
|
||||
@@ -6,6 +6,7 @@ from dataclasses import replace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Glm5NextTextConfig
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
@@ -44,7 +45,6 @@ from vllm.model_executor.utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
@@ -201,7 +201,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Glm5NextConfig,
|
||||
config: Glm5NextTextConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import ClassVar, Literal
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Glm5NextTextConfig
|
||||
|
||||
from vllm.config import ParallelConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
@@ -83,7 +84,6 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig
|
||||
|
||||
from .attention import Glm5NextMLAAttention
|
||||
from .kda import Glm5NextLinearAttention
|
||||
@@ -95,6 +95,57 @@ from .multimodal import (
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_MHC_TAU = 0.05
|
||||
"""mHC routing temperature. A GLM-5.3-Flash trained value that neither the
|
||||
checkpoint nor `Glm5NextTextConfig` carries."""
|
||||
|
||||
_MHC_POST_MULT_VALUE = 2.0
|
||||
"""mHC post-multiplier. A GLM-5.3-Flash trained value that neither the
|
||||
checkpoint nor `Glm5NextTextConfig` carries."""
|
||||
|
||||
_LOGIT_SCALE = 1.0
|
||||
"""Output logit scale. A GLM-5.3-Flash trained value that neither the
|
||||
checkpoint nor `Glm5NextTextConfig` carries."""
|
||||
|
||||
_VISION_RMS_NORM_EPS = 1e-6
|
||||
"""Vision tower RMSNorm epsilon.
|
||||
|
||||
GLM-5.3-Flash checkpoints ship `vision_config.rms_norm_eps = 1e-5`, but the
|
||||
vision tower was trained with 1e-6. Serving with 1e-5 drifts the RMSNorm and
|
||||
produces repetitive/degraded image descriptions, so force the trained value
|
||||
regardless of the checkpoint field.
|
||||
"""
|
||||
|
||||
|
||||
def _is_moe(config: Glm5NextTextConfig) -> bool:
|
||||
return config.n_routed_experts is not None
|
||||
|
||||
|
||||
def _is_kda_layer(config: Glm5NextTextConfig, layer_idx: int) -> bool:
|
||||
layer_types = config.layer_types
|
||||
return (
|
||||
layer_types is not None
|
||||
and layer_idx < len(layer_types)
|
||||
and layer_types[layer_idx] == "linear_attention"
|
||||
)
|
||||
|
||||
|
||||
def _is_linear_attn(config: Glm5NextTextConfig) -> bool:
|
||||
layer_types = config.layer_types
|
||||
return layer_types is not None and "linear_attention" in layer_types
|
||||
|
||||
|
||||
def _validate_supported_config(config: Glm5NextTextConfig) -> None:
|
||||
"""Reject checkpoints using config options this implementation lacks.
|
||||
|
||||
The kpool indexer kernels always keep the incomplete trailing pool, so a
|
||||
checkpoint asking otherwise would be served silently wrong.
|
||||
"""
|
||||
if config.index_topk is not None and not config.index_kpool_always_select_tail:
|
||||
raise NotImplementedError(
|
||||
"GLM-5.3 sparse indexer requires index_kpool_always_select_tail=True"
|
||||
)
|
||||
|
||||
|
||||
class Glm5NextMLP(nn.Module):
|
||||
def __init__(
|
||||
@@ -152,7 +203,7 @@ class Glm5NextMLP(nn.Module):
|
||||
class Glm5NextMoE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: Glm5NextConfig,
|
||||
config: Glm5NextTextConfig,
|
||||
parallel_config: ParallelConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
@@ -227,10 +278,10 @@ class Glm5NextMoE(nn.Module):
|
||||
shared_experts=self.shared_experts,
|
||||
gate=self.gate,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_token,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
renormalize=config.moe_renormalize,
|
||||
renormalize=config.norm_topk_prob,
|
||||
quant_config=quant_config,
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=config.n_group,
|
||||
@@ -280,7 +331,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: Glm5NextConfig,
|
||||
config: Glm5NextTextConfig,
|
||||
layer_idx: int,
|
||||
prefix: str = "",
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
@@ -295,13 +346,13 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
|
||||
self.hidden_size = config.hidden_size
|
||||
self.layer_idx = layer_idx
|
||||
self.is_moe = config.is_moe
|
||||
self.is_moe = _is_moe(config)
|
||||
self.num_hidden_layers = config.num_hidden_layers
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.num_experts = config.n_routed_experts
|
||||
self.is_mtp_layer = is_mtp_layer
|
||||
self.mhc = config.mhc
|
||||
is_kda_layer = not is_mtp_layer and config.is_kda_layer(layer_idx)
|
||||
is_kda_layer = not is_mtp_layer and _is_kda_layer(config, layer_idx)
|
||||
self.layer_kind = "kda" if is_kda_layer else "mla"
|
||||
self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe
|
||||
|
||||
@@ -331,7 +382,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
quant_config=None, # MLA projections are BF16 in checkpoint
|
||||
prefix=f"{prefix}.self_attn",
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
skip_rope=config.mla_nope,
|
||||
skip_rope=config.mla_use_nope,
|
||||
)
|
||||
|
||||
# MTP layers sit past the base model's hidden layers (layer_idx >=
|
||||
@@ -375,13 +426,13 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
|
||||
if self.mhc and not is_mtp_layer:
|
||||
# mhc config
|
||||
self.mhc_num_residual_streams = config.mhc_num_residual_streams
|
||||
self.mhc_tau = config.mhc_tau
|
||||
self.mhc_num_residual_streams = config.hc_mult
|
||||
self.mhc_tau = _MHC_TAU
|
||||
self.hc_eps = config.hc_eps
|
||||
self.mhc_sinkhorn_iterations = config.mhc_sinkhorn_iterations
|
||||
self.mhc_post_mult_value = config.mhc_post_mult_value
|
||||
self.mhc_sinkhorn_iterations = config.hc_sinkhorn_iters
|
||||
self.mhc_post_mult_value = _MHC_POST_MULT_VALUE
|
||||
|
||||
n = config.mhc_num_residual_streams
|
||||
n = config.hc_mult
|
||||
d_model = n * self.hidden_size
|
||||
mix_hc = (2 + n) * n
|
||||
|
||||
@@ -625,7 +676,8 @@ class Glm5NextModel(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
_validate_supported_config(config)
|
||||
self.config = config
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
@@ -768,7 +820,7 @@ class Glm5NextModel(nn.Module):
|
||||
(".in_proj_qkvbfg_a", ".f_a_proj", 4),
|
||||
(".in_proj_qkvbfg_a", ".g_a_proj", 5),
|
||||
]
|
||||
if self.config.is_moe:
|
||||
if _is_moe(self.config):
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
# EPLB: the mapping enumerates physical experts, so it must cover
|
||||
@@ -798,7 +850,7 @@ class Glm5NextModel(nn.Module):
|
||||
# GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from
|
||||
# ``kv_a_proj_with_mqa``; pad them with zeros for the model shape.
|
||||
kv_a_pad_size = 0
|
||||
if self.config.mla_nope and self.config.qk_rope_head_dim > 0:
|
||||
if self.config.mla_use_nope and self.config.qk_rope_head_dim > 0:
|
||||
kv_a_pad_size = self.config.qk_rope_head_dim
|
||||
|
||||
_pending_wk_fp8: dict = {}
|
||||
@@ -912,7 +964,7 @@ class Glm5NextModel(nn.Module):
|
||||
if (
|
||||
name.endswith(".bias")
|
||||
and name not in params_dict
|
||||
and not self.config.is_linear_attn
|
||||
and not _is_linear_attn(self.config)
|
||||
): # noqa: E501
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
@@ -939,7 +991,7 @@ class Glm5NextForCausalLM(
|
||||
super().__init__()
|
||||
self.model_config = vllm_config.model_config
|
||||
self.vllm_config = vllm_config
|
||||
self.config = self.model_config.hf_config
|
||||
self.config = self.model_config.hf_text_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.quant_config = quant_config
|
||||
self.model = Glm5NextModel(
|
||||
@@ -955,7 +1007,7 @@ class Glm5NextForCausalLM(
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.config.vocab_size, scale=self.config.logit_scale
|
||||
self.config.vocab_size, scale=_LOGIT_SCALE
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
@@ -988,7 +1040,7 @@ class Glm5NextForCausalLM(
|
||||
cls, vllm_config: "VllmConfig"
|
||||
) -> tuple[tuple[int, int], tuple[int, int, int]]:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
hf_config = vllm_config.model_config.hf_text_config
|
||||
tp_size = parallel_config.tensor_parallel_size
|
||||
num_spec = (
|
||||
vllm_config.speculative_config.num_speculative_tokens
|
||||
@@ -1092,11 +1144,7 @@ class Glm5NextForConditionalGeneration(
|
||||
self.visual = Glm5NextVisionTransformer(
|
||||
config.text_config,
|
||||
config.vision_config,
|
||||
# Read eps from the VISION sub-config, not the top-level
|
||||
# `config.rms_norm_eps`: Glm5NextConfig.__getattribute__ mirrors
|
||||
# the latter onto text_config (1e-5), silently ignoring the
|
||||
# vision tower's own (1e-6) rms_norm_eps.
|
||||
norm_eps=config.vision_config.rms_norm_eps,
|
||||
norm_eps=_VISION_RMS_NORM_EPS,
|
||||
# Vision tower ships BF16 weights in this fp8 checkpoint (no
|
||||
# weight_scale_inv for visual.*), so it must NOT inherit the
|
||||
# global fp8 quant_config -- doing so incorrectly quantizes
|
||||
@@ -1166,7 +1214,7 @@ class Glm5NextForConditionalGeneration(
|
||||
|
||||
|
||||
def get_spec_layer_idx_from_weight_name(
|
||||
config: Glm5NextConfig, weight_name: str
|
||||
config: Glm5NextTextConfig, weight_name: str
|
||||
) -> int | None:
|
||||
if hasattr(config, "num_nextn_predict_layers") and (
|
||||
config.num_nextn_predict_layers > 0
|
||||
|
||||
@@ -40,7 +40,7 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_text_config
|
||||
self.config = config
|
||||
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -114,7 +114,7 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module):
|
||||
class Glm5NextMultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
self.mtp_start_layer_idx = config.num_hidden_layers
|
||||
self.num_mtp_layers = config.num_nextn_predict_layers
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
@@ -211,7 +211,7 @@ class Glm5NextMultiTokenPredictor(nn.Module):
|
||||
class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.config = vllm_config.model_config.hf_text_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.model = Glm5NextMultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
@@ -313,7 +313,7 @@ class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
# GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from
|
||||
# ``kv_a_proj_with_mqa``; the FP8-to-BF16 path pads them for the model.
|
||||
kv_a_pad_size = 0
|
||||
if self.config.mla_nope and self.config.qk_rope_head_dim > 0:
|
||||
if self.config.mla_use_nope and self.config.qk_rope_head_dim > 0:
|
||||
kv_a_pad_size = self.config.qk_rope_head_dim
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
|
||||
@@ -3,13 +3,14 @@
|
||||
"""GLM-5.3-Flash vision tower and multimodal processor."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from functools import cached_property, partial
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -44,7 +45,6 @@ from vllm.model_executor.models.vision import (
|
||||
)
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
||||
from vllm.transformers_utils.processors.glm5next import glm_sample_frame_indices
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
@@ -616,37 +616,48 @@ class Glm5NextVisionTransformer(nn.Module):
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
"""Wires up the vLLM-native processor for the multimodal checkpoint.
|
||||
_MAX_VIDEO_TOKENS = 30000
|
||||
|
||||
The checkpoint's ``processor_config.json`` declares a custom ``processor_class``
|
||||
and stores its image/video processor configs inline (no standalone
|
||||
``preprocessor_config.json``), so ``AutoProcessor`` cannot resolve the
|
||||
config. We bypass it and build our own ``Glm5NextProcessor``
|
||||
(``vllm/transformers_utils/processors/glm5next.py``), a port of the
|
||||
training-side pipeline that no longer imports transformers' GLM processor
|
||||
classes. The port applies ``patch_expand_factor`` (checkpoint ships 1)
|
||||
inside ``smart_resize``'s spatial factor.
|
||||
|
||||
class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
"""Token-budget geometry for the multimodal checkpoint.
|
||||
|
||||
This checkpoint's ``processor_config.json`` ships the token-budget style
|
||||
(``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, so the
|
||||
inherited Glm4v ``size.longest_edge`` path does not apply.
|
||||
"""
|
||||
|
||||
@cached_property
|
||||
def _glm5_hf_processor(self):
|
||||
from vllm.transformers_utils.processors.glm5next import Glm5NextProcessor
|
||||
|
||||
return Glm5NextProcessor.from_pretrained(self.ctx.model_config.model)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
return self._glm5_hf_processor
|
||||
processor = super().get_hf_processor(**kwargs)
|
||||
# Cap video inputs to keep encoder profiling from starving the KV cache.
|
||||
video_processor = processor.video_processor
|
||||
video_processor.max_image_tokens = min(
|
||||
video_processor.max_image_tokens, _MAX_VIDEO_TOKENS
|
||||
)
|
||||
return processor
|
||||
|
||||
@staticmethod
|
||||
def _alignment_factor(proc) -> int:
|
||||
"""Spatial factor canvases align to, as ``smart_resize`` takes it."""
|
||||
return proc.patch_size * proc.merge_size * proc.patch_expand_factor
|
||||
|
||||
@classmethod
|
||||
def _pixels_per_token(cls, proc) -> int:
|
||||
"""Pixels one vision token covers, matching ``smart_resize``'s own
|
||||
``temporal_factor * factor ** 2``."""
|
||||
return proc.temporal_patch_size * cls._alignment_factor(proc) ** 2
|
||||
|
||||
def _processor_pixel_budget(self, proc) -> tuple[int, int]:
|
||||
from vllm.transformers_utils.processors.glm5next import _pixel_budget
|
||||
|
||||
return _pixel_budget(
|
||||
proc.min_image_tokens,
|
||||
proc.max_image_tokens,
|
||||
proc.patch_size,
|
||||
proc.merge_size,
|
||||
proc.temporal_patch_size,
|
||||
"""(min_pixels, max_pixels) from the processor's token bounds."""
|
||||
if proc.min_image_tokens is None or proc.max_image_tokens is None:
|
||||
raise ValueError(
|
||||
"min_image_tokens and max_image_tokens must be provided by "
|
||||
"processor_config.json (or per-call kwargs)."
|
||||
)
|
||||
pixels_per_token = self._pixels_per_token(proc)
|
||||
return (
|
||||
proc.min_image_tokens * pixels_per_token,
|
||||
proc.max_image_tokens * pixels_per_token,
|
||||
)
|
||||
|
||||
def _get_image_max_pixels(self) -> int:
|
||||
@@ -687,19 +698,20 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
else:
|
||||
frame_indices = [
|
||||
int(idx)
|
||||
for idx in glm_sample_frame_indices(
|
||||
int(metadata.get("total_num_frames", total_frames)),
|
||||
video_fps,
|
||||
float(metadata.get("duration") or 0),
|
||||
target_fps=video_processor.fps_interval,
|
||||
max_frame_count=video_processor.max_frame_count_dynamic,
|
||||
temporal_patch_size=temporal_patch_size,
|
||||
for idx in video_processor.sample_frames(
|
||||
VideoMetadata(
|
||||
total_num_frames=int(
|
||||
metadata.get("total_num_frames", total_frames)
|
||||
),
|
||||
fps=video_fps,
|
||||
duration=metadata.get("duration"),
|
||||
)
|
||||
)
|
||||
]
|
||||
num_frames = len(frame_indices)
|
||||
|
||||
if not frame_indices or num_frames <= 0:
|
||||
# ``duration * fps_interval < 1`` selects no frames at all; the
|
||||
# ``duration * fps < 1`` selects no frames at all; the
|
||||
# pixel path raises an opaque IndexError on the same input a moment
|
||||
# later, so say what happened while there is still a request.
|
||||
raise ValueError(
|
||||
@@ -731,12 +743,13 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
|
||||
The inherited Glm4v path resolves the pixel budget from
|
||||
``size.longest_edge`` and resizes with GLM-4V's ``smart_resize``. This
|
||||
checkpoint's ``processor_config.json`` ships the token-budget style
|
||||
(``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, and
|
||||
the alignment factor carries ``patch_expand_factor`` — resolve both
|
||||
from the vLLM-native processor so profiling matches runtime geometry.
|
||||
checkpoint ships token bounds instead, and the alignment factor carries
|
||||
``patch_expand_factor`` — resolve both from the processor so profiling
|
||||
matches runtime geometry.
|
||||
"""
|
||||
from vllm.transformers_utils.processors.glm5next import smart_resize
|
||||
from transformers.models.glm5_next.image_processing_glm5_next import (
|
||||
smart_resize,
|
||||
)
|
||||
|
||||
vision_config = self.get_hf_config().vision_config
|
||||
patch_size = vision_config.patch_size
|
||||
@@ -744,22 +757,25 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
temporal_patch_size = vision_config.temporal_patch_size
|
||||
|
||||
image_processor = self.get_hf_processor().image_processor
|
||||
factor = patch_size * merge_size * image_processor.patch_expand_factor
|
||||
# Keep the profiling search viable when the caller's budget is below
|
||||
# one aligned canvas of the requested duration.
|
||||
max_image_pixels = max(max_image_pixels, temporal_patch_size * factor * factor)
|
||||
factor = self._alignment_factor(image_processor)
|
||||
temporal_factor = image_processor.temporal_patch_size
|
||||
# `smart_resize` denominates its bounds in vision tokens. Round down,
|
||||
# but keep the profiling search viable when the caller's budget is
|
||||
# below one aligned canvas of the requested duration.
|
||||
max_image_tokens = max(
|
||||
max_image_pixels // self._pixels_per_token(image_processor), 1
|
||||
)
|
||||
|
||||
if do_resize:
|
||||
t = num_frames if num_frames > temporal_patch_size else temporal_patch_size
|
||||
t = num_frames if num_frames > temporal_factor else temporal_factor
|
||||
resized_height, resized_width = smart_resize(
|
||||
t=t,
|
||||
h=image_height,
|
||||
w=image_width,
|
||||
t_factor=temporal_patch_size,
|
||||
h_factor=factor,
|
||||
w_factor=factor,
|
||||
num_frames=t,
|
||||
height=image_height,
|
||||
width=image_width,
|
||||
temporal_factor=temporal_factor,
|
||||
factor=factor,
|
||||
min_pixels=1,
|
||||
max_pixels=max_image_pixels,
|
||||
max_pixels=max_image_tokens,
|
||||
)
|
||||
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
|
||||
else:
|
||||
@@ -777,7 +793,7 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo):
|
||||
|
||||
|
||||
class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor):
|
||||
"""The vLLM-native ``Glm5NextProcessor`` extracts image/video features
|
||||
"""``Glm5NextProcessor`` extracts image/video features
|
||||
only and passes the prompt text through unchanged, so prompt expansion
|
||||
(image token repeat, video frame/timestamp structure) is owned by vLLM's
|
||||
prompt-update machinery — the inherited ``_get_prompt_updates`` builds
|
||||
|
||||
Reference in New Issue
Block a user