From 38de822310239a50c610a420e2ed9f6dd670d5a7 Mon Sep 17 00:00:00 2001 From: Terry Gao <32590313+tianrengao@users.noreply.github.com> Date: Wed, 25 Mar 2026 23:52:29 -0700 Subject: [PATCH] [Model] Add torch.compile support for InternVL vision encoder (#38049) Signed-off-by: tianrengao --- vllm/config/utils.py | 10 +++++++++- vllm/model_executor/models/intern_vit.py | 13 +++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/vllm/config/utils.py b/vllm/config/utils.py index 8996660483..73abd78656 100644 --- a/vllm/config/utils.py +++ b/vllm/config/utils.py @@ -296,7 +296,15 @@ def normalize_value(x): # PretrainedConfig if hasattr(x, "to_json_string") and callable(x.to_json_string): - return x.to_json_string() + try: + return x.to_json_string() + except (TypeError, ValueError): + # to_json_string() may fail for trust-remote-code configs + # with non-JSON-serializable nested objects. Fall back to + # normalizing the dict representation recursively. + if hasattr(x, "to_dict") and callable(x.to_dict): + return normalize_value(x.to_dict()) + raise # Unsupported type: e.g., modules, generators, open files, or objects # without a stable JSON/UUID representation. Hard-error to avoid diff --git a/vllm/model_executor/models/intern_vit.py b/vllm/model_executor/models/intern_vit.py index 8161473641..7da498bf6d 100644 --- a/vllm/model_executor/models/intern_vit.py +++ b/vllm/model_executor/models/intern_vit.py @@ -15,6 +15,10 @@ import torch.nn as nn import torch.nn.functional as F from transformers import PretrainedConfig +from vllm.compilation.decorators import ( + should_torch_compile_mm_encoder, + support_torch_compile, +) from vllm.distributed import ( divide, get_tensor_model_parallel_rank, @@ -280,6 +284,11 @@ class InternMLP(nn.Module): return hidden_states +@support_torch_compile( + dynamic_arg_dims={"hidden_states": 0}, + enable_if=should_torch_compile_mm_encoder, + is_encoder=True, +) class InternVisionEncoderLayer(nn.Module): def __init__( self, @@ -364,8 +373,8 @@ class InternVisionEncoder(nn.Module): self.layers = nn.ModuleList( [ self.layer_cls( - config, - quant_config, + config=config, + quant_config=quant_config, num_dummy_heads=num_dummy_heads, prefix=f"{prefix}.layers.{layer_idx}", )