[Bugfix][Gemma4] Quantize audio towers independently (#59935)

Signed-off-by: sudhanshu shukla <sudhanshu112233shukla@users.noreply.github.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: sudhanshu shukla <sudhanshu112233shukla@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
sudhanshu shukla
2026-10-14 11:27:44 +00:00
committed by GitHub
co-authored by sudhanshu shukla Isotr0py
parent 23fbc2ec2f
commit d39c2fb05a
+32 -23
View File
@@ -98,6 +98,28 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _get_tower_quant_config(
tower_config: Any,
quant_config: "QuantizationConfig | None",
) -> "QuantizationConfig | None":
# BNB/torchao/compressed-tensors handle arbitrary sizes; other methods
# (Marlin, FP8, …) require dimensions divisible by 64.
# TODO(mgoin): remove this by fixing kernel padding.
if quant_config and quant_config.get_name() in [
"bitsandbytes",
"torchao",
"compressed-tensors",
]:
return quant_config
# The audio config has no intermediate_size; treat it as always divisible.
intermediate_size = getattr(tower_config, "intermediate_size", None)
quantizable = tower_config.hidden_size % 64 == 0 and (
intermediate_size is None or intermediate_size % 64 == 0
)
return quant_config if quantizable else None
# Video constants — match transformers Gemma4VideoProcessor defaults.
_SUPPORTED_SOFT_TOKENS = (70, 140, 280, 560, 1120)
_VIDEO_MAX_SOFT_TOKENS = 70 # soft tokens per video frame (vs 280 for images)
@@ -1071,25 +1093,9 @@ class Gemma4ForConditionalGeneration(
lora_config is not None and lora_config.enable_tower_connector_lora
)
# Only quantize towers when the quant method supports their
# dimensions. BNB/torchao handle arbitrary sizes; other methods
# (Marlin, FP8, …) require dimensions divisible by 64, which
# the vision tower (intermediate_size=4304) does not satisfy.
# TODO(mgoin): remove this by fixing kernel padding.
tower_quant: QuantizationConfig | None
if quant_config and quant_config.get_name() in [
"bitsandbytes",
"torchao",
"compressed-tensors",
]:
tower_quant = quant_config
else:
vision_cfg = config.vision_config
quantizable = (
vision_cfg.hidden_size % 64 == 0
and vision_cfg.intermediate_size % 64 == 0
)
tower_quant = quant_config if quantizable else None
# Apply the dimension check independently because the vision and audio
# towers do not necessarily have the same hidden and intermediate sizes.
vision_tower_quant = _get_tower_quant_config(config.vision_config, quant_config)
# ---- Vision tower (shared by image and video) ----
with self._mark_tower_model(vllm_config, {"image", "video"}):
@@ -1097,18 +1103,21 @@ class Gemma4ForConditionalGeneration(
self.embed_vision = Gemma4MultimodalEmbedder(
config.vision_config,
config.text_config,
quant_config=tower_quant,
quant_config=vision_tower_quant,
prefix=maybe_prefix(prefix, "embed_vision"),
)
recursive_replace_linear(
self.vision_tower,
tower_quant,
vision_tower_quant,
prefix=maybe_prefix(prefix, "vision_tower"),
)
# ---- Audio tower (variants with audio_config) ----
self.embed_audio: Gemma4MultimodalEmbedder | None
if config.audio_config is not None:
audio_tower_quant = _get_tower_quant_config(
config.audio_config, quant_config
)
with self._mark_tower_model(vllm_config, "audio"):
self.audio_tower = AutoModel.from_config(config=config.audio_config)
# AutoModel.from_config does NOT call post_init(),
@@ -1119,12 +1128,12 @@ class Gemma4ForConditionalGeneration(
self.embed_audio = Gemma4MultimodalEmbedder(
config.audio_config,
config.text_config,
quant_config=tower_quant,
quant_config=audio_tower_quant,
prefix=maybe_prefix(prefix, "embed_audio"),
)
recursive_replace_linear(
self.audio_tower,
tower_quant,
audio_tower_quant,
prefix=maybe_prefix(prefix, "audio_tower"),
)
else: