[Chore] Remove vendored code made redundant by Transformers 5.16.1 (#60623)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-10-09 09:29:03 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent bc713cf8cc
commit 2c99ee9333
76 changed files with 719 additions and 3268 deletions
+42 -54
View File
@@ -493,13 +493,9 @@ class Granite4VisionForConditionalGeneration(
prefix=maybe_prefix(prefix, "vision_tower"),
)
# image_newline parameter
if config.use_image_newline_parameter:
self.image_newline = nn.Parameter(
torch.empty(config.text_config.hidden_size)
)
else:
self.image_newline = None
self.image_newline = nn.Parameter(
torch.empty(config.text_config.hidden_size)
)
cache_config = vllm_config.cache_config
@@ -517,20 +513,18 @@ class Granite4VisionForConditionalGeneration(
)
# Spatial projectors: 4 offset groups
self.spatial_projectors = None
if config.use_spatial_sampling:
self.spatial_projectors = nn.ModuleList(
[
WindowQFormerDownsampler(
config,
quant_config=quant_config,
cache_config=cache_config,
spatial_offset=i,
prefix=maybe_prefix(prefix, f"spatial_projectors.{i}"),
)
for i in range(4)
]
)
self.spatial_projectors = nn.ModuleList(
[
WindowQFormerDownsampler(
config,
quant_config=quant_config,
cache_config=cache_config,
spatial_offset=i,
prefix=maybe_prefix(prefix, f"spatial_projectors.{i}"),
)
for i in range(4)
]
)
# ----- Language model (marked as LM) -----
with self._mark_language_model(vllm_config):
@@ -545,12 +539,9 @@ class Granite4VisionForConditionalGeneration(
# Store config values we need
self._deepstack_layer_map = config.deepstack_layer_map # [[-19, 9], ...]
self._use_spatial_sampling = getattr(config, "use_spatial_sampling", False)
self._spatial_vision_layer = getattr(config, "spatial_vision_layer", -1)
self._spatial_target_layers = getattr(config, "spatial_target_layers", [])
self._vision_feature_select_strategy = getattr(
config, "vision_feature_select_strategy", "full"
)
self._spatial_vision_layer = config.spatial_vision_layer
self._spatial_target_layers = config.spatial_target_layers
self._vision_feature_select_strategy = config.vision_feature_select_strategy
self._downsample_rate = Fraction(config.downsample_rate)
# Ordered list of LLM layer indices for each deepstack level.
@@ -558,7 +549,7 @@ class Granite4VisionForConditionalGeneration(
# (before any embed_multimodal call).
self._ds_layer_indices: list[int] = [
llm_layer for _, llm_layer in config.deepstack_layer_map
] + list(getattr(config, "spatial_target_layers", []))
] + list(config.spatial_target_layers)
# Share ds_layer_indices with the LLM causal model so
# make_empty_intermediate_tensors includes the correct keys
@@ -571,7 +562,7 @@ class Granite4VisionForConditionalGeneration(
# the updated values written just before each prefill.
# Shape: (max_num_batched_tokens, lm_hidden_size) per level.
n_layerwise = len(config.deepstack_layer_map)
n_spatial = len(getattr(config, "spatial_target_layers", []))
n_spatial = len(config.spatial_target_layers)
num_ds_levels = n_layerwise + n_spatial
lm_hidden = config.text_config.hidden_size
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
@@ -644,26 +635,24 @@ class Granite4VisionForConditionalGeneration(
)
image_feature = unpad_image(image_feature, image_sizes[image_idx])
if self.image_newline is not None:
image_feature = torch.cat(
(
image_feature,
self.image_newline[:, None, None]
.expand(*image_feature.shape[:-1], 1)
.to(image_feature.device, image_feature.dtype),
),
dim=-1,
)
image_feature = torch.cat(
(
image_feature,
self.image_newline[:, None, None]
.expand(*image_feature.shape[:-1], 1)
.to(image_feature.device, image_feature.dtype),
),
dim=-1,
)
image_feature = image_feature.flatten(1, 2).transpose(0, 1)
image_feature = torch.cat((base_image_feature, image_feature), dim=0)
else:
image_feature = image_feature[0]
if self.image_newline is not None:
image_feature = torch.cat(
(image_feature, self.image_newline[None].to(image_feature)),
dim=0,
)
image_feature = torch.cat(
(image_feature, self.image_newline[None].to(image_feature)),
dim=0,
)
new_image_features.append(image_feature)
@@ -718,16 +707,15 @@ class Granite4VisionForConditionalGeneration(
)
levels.append((llm_layer, per_image))
if self._use_spatial_sampling and self.spatial_projectors is not None:
spatial_hidden = all_hidden_states[self._spatial_vision_layer]
if select_strategy == "default":
spatial_hidden = spatial_hidden[:, 1:]
for group_idx, llm_layer in enumerate(self._spatial_target_layers):
projected = self.spatial_projectors[group_idx](spatial_hidden)
per_image = self._pack_and_unpad_image_features(
torch.split(projected, image_num_patches, dim=0), image_sizes
)
levels.append((llm_layer, per_image))
spatial_hidden = all_hidden_states[self._spatial_vision_layer]
if select_strategy == "default":
spatial_hidden = spatial_hidden[:, 1:]
for group_idx, llm_layer in enumerate(self._spatial_target_layers):
projected = self.spatial_projectors[group_idx](spatial_hidden)
per_image = self._pack_and_unpad_image_features(
torch.split(projected, image_num_patches, dim=0), image_sizes
)
levels.append((llm_layer, per_image))
llm_layer_indices = [llm_layer for llm_layer, _ in levels]
num_images = len(image_sizes)