[Chore] Remove use_data_parallel kwargs from ViT implementation (#33310)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2026-01-29 10:20:52 +00:00
committed by GitHub
parent 3a92c6f3b5
commit 5400014d55
9 changed files with 36 additions and 89 deletions
+10 -25
View File
@@ -34,7 +34,7 @@ from vllm.model_executor.layers.linear import (
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from .vision import run_dp_sharded_vision_model
from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model
NORM2FN = {
"rms_norm": RMSNorm,
@@ -148,7 +148,6 @@ class InternParallelAttention(nn.Module):
*,
num_dummy_heads: int = 0,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
super().__init__()
@@ -163,9 +162,14 @@ class InternParallelAttention(nn.Module):
f" {self.num_heads})."
)
self.tp_size = (
1 if use_data_parallel else get_tensor_model_parallel_world_size()
use_data_parallel = is_vit_use_data_parallel()
# if the number of heads is not divisible by tp_size,
# we also disable Attention's TP
tp_size = 1 if use_data_parallel else get_tensor_model_parallel_world_size()
use_data_parallel = (
use_data_parallel or (self.num_heads + num_dummy_heads) % tp_size != 0
)
self.tp_size = 1 if use_data_parallel else tp_size
self.tp_rank = 0 if use_data_parallel else get_tensor_model_parallel_rank()
# Additional dummy heads are used to enable TP for common GPU counts.
@@ -242,12 +246,12 @@ class InternMLP(nn.Module):
config: PretrainedConfig,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
super().__init__()
self.config = config
self.activation_fn = get_act_fn(config.hidden_act)
use_data_parallel = is_vit_use_data_parallel()
self.fc1 = ColumnParallelLinear(
config.hidden_size,
config.intermediate_size,
@@ -281,11 +285,9 @@ class InternVisionEncoderLayer(nn.Module):
*,
num_dummy_heads: int = 0,
prefix: str = "",
use_data_parallel: bool = False,
attn_cls: type[InternParallelAttention] = InternParallelAttention,
) -> None:
super().__init__()
self.embed_dim = config.hidden_size
self.intermediate_size = config.intermediate_size
self.norm_type = config.norm_type
@@ -296,14 +298,12 @@ class InternVisionEncoderLayer(nn.Module):
quant_config,
num_dummy_heads=num_dummy_heads,
prefix=f"{prefix}.attn",
use_data_parallel=use_data_parallel,
)
self.mlp = InternMLP(
config,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
use_data_parallel=use_data_parallel,
)
self.norm1 = NORM2FN[self.norm_type](self.embed_dim, eps=config.layer_norm_eps)
self.norm2 = NORM2FN[self.norm_type](self.embed_dim, eps=config.layer_norm_eps)
@@ -318,23 +318,12 @@ class InternVisionEncoderLayer(nn.Module):
*,
num_dummy_heads: int,
prefix: str = "",
use_data_parallel: bool = False,
):
# fallback to sdpa attention if tp unavailable
tp_size = 1 if use_data_parallel else get_tensor_model_parallel_world_size()
num_heads = config.num_attention_heads
# if the number of heads is not divisible by tp_size,
# we also disable Attention's TP
use_data_parallel = (
use_data_parallel or (num_heads + num_dummy_heads) % tp_size != 0
)
return self.attn_cls(
config,
quant_config=quant_config,
num_dummy_heads=num_dummy_heads,
prefix=prefix,
use_data_parallel=use_data_parallel,
)
def forward(
@@ -357,7 +346,6 @@ class InternVisionEncoder(nn.Module):
num_hidden_layers_override: int | None = None,
num_dummy_heads: int = 0,
prefix: str = "",
use_data_parallel: bool = False,
layer_cls: type[InternVisionEncoderLayer] = InternVisionEncoderLayer,
):
super().__init__()
@@ -377,7 +365,6 @@ class InternVisionEncoder(nn.Module):
quant_config,
num_dummy_heads=num_dummy_heads,
prefix=f"{prefix}.layers.{layer_idx}",
use_data_parallel=use_data_parallel,
)
for layer_idx in range(num_hidden_layers)
]
@@ -404,12 +391,11 @@ class InternVisionModel(nn.Module):
num_hidden_layers_override: int | None = None,
num_dummy_heads: int = 0,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
super().__init__()
self.config = config
self.use_data_parallel = use_data_parallel
self.use_data_parallel = is_vit_use_data_parallel()
self.embeddings = InternVisionEmbeddings(config)
self.encoder = InternVisionEncoder(
@@ -418,7 +404,6 @@ class InternVisionModel(nn.Module):
num_hidden_layers_override=num_hidden_layers_override,
num_dummy_heads=num_dummy_heads,
prefix=f"{prefix}.encoder",
use_data_parallel=use_data_parallel,
)
def get_input_embeddings(self):