[Quant] Support compressed-tensors WNA8O8Int linears and WNInt embeddings (#44340)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-06-04 07:40:33 -07:00
committed by GitHub
parent b5235fca2e
commit 06ee2d8433
14 changed files with 744 additions and 27 deletions
+3 -3
View File
@@ -1057,7 +1057,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
"normalizer",
torch.tensor(
config.hidden_size**0.5,
dtype=self.embed_tokens.weight.dtype,
dtype=vllm_config.model_config.dtype,
),
persistent=False,
)
@@ -1111,7 +1111,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
)
self.hidden_states = torch.zeros(
(max_num_tokens, config.hidden_size),
dtype=self.embed_tokens.weight.dtype,
dtype=vllm_config.model_config.dtype,
device=device,
)
if (
@@ -1124,7 +1124,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
config.num_hidden_layers,
self.hidden_size_per_layer_input,
),
dtype=self.embed_tokens.weight.dtype,
dtype=vllm_config.model_config.dtype,
device=device,
)
else:
+12 -12
View File
@@ -1011,14 +1011,17 @@ class Gemma4ForConditionalGeneration(
self.config = config
self.quant_config = quant_config
self.multimodal_config = multimodal_config
self.model_dtype = vllm_config.model_config.dtype
# Only quantize towers when the quant method supports their
# dimensions. BNB/torchao handle arbitrary sizes; other methods
# (Marlin, FP8, …) require dimensions divisible by 64, which
# the vision tower (intermediate_size=4304) does not satisfy.
# TODO(mgoin): remove this by fixing kernel padding.
if quant_config and quant_config.get_name() in [
"bitsandbytes",
"torchao",
"compressed-tensors",
]:
tower_quant = quant_config
else:
@@ -1081,12 +1084,13 @@ class Gemma4ForConditionalGeneration(
# Some variants have hidden_size_per_layer_input=None (no PLE).
ple_dim = config.text_config.hidden_size_per_layer_input
if ple_dim is not None and ple_dim > 0:
embed = self.language_model.model.embed_tokens
self.per_layer_embeddings = torch.zeros(
vllm_config.scheduler_config.max_num_batched_tokens,
config.text_config.num_hidden_layers,
ple_dim,
device=self.language_model.model.embed_tokens.weight.device,
dtype=self.language_model.model.embed_tokens.weight.dtype,
device=next(embed.parameters()).device,
dtype=vllm_config.model_config.dtype,
)
else:
self.per_layer_embeddings = None
@@ -1246,7 +1250,6 @@ class Gemma4ForConditionalGeneration(
vt = self.vision_tower
vision_cfg = self.config.vision_config
pooling_k2 = vision_cfg.pooling_kernel_size**2
target_dtype = self.language_model.model.embed_tokens.weight.dtype
# Concurrent requests with different image resolutions may
# arrive as a list of per-image tensors, while same-resolution
@@ -1291,7 +1294,7 @@ class Gemma4ForConditionalGeneration(
pv_tensor,
pp_tensor,
pad_tensor,
).to(target_dtype)
).to(self.model_dtype)
encoder_outputs = vt.encoder(
inputs_embeds=inputs_embeds,
attention_mask=~pad_tensor,
@@ -1328,12 +1331,8 @@ class Gemma4ForConditionalGeneration(
all_valid_states[orig_idx] = valid_states
valid_lens[orig_idx] = valid_states.shape[0]
# Use embed_tokens dtype as compute dtype; embedding_projection.weight
# may be uint8 under BnB 4-bit, which would corrupt the cast.
target_dtype = self.language_model.model.embed_tokens.weight.dtype
# Project all images in a single batched call.
flat_valid_states = torch.cat(all_valid_states, dim=0).to(target_dtype)
flat_valid_states = torch.cat(all_valid_states, dim=0).to(self.model_dtype)
flat_proj_embs = self.embed_vision(
inputs_embeds=flat_valid_states.unsqueeze(0)
).squeeze(0)
@@ -1373,7 +1372,6 @@ class Gemma4ForConditionalGeneration(
vt = self.vision_tower
vision_cfg = self.config.vision_config
pooling_k2 = vision_cfg.pooling_kernel_size**2
target_dtype = self.language_model.model.embed_tokens.weight.dtype
if isinstance(frame_counts, torch.Tensor):
fc_list = frame_counts.tolist()
@@ -1405,7 +1403,7 @@ class Gemma4ForConditionalGeneration(
pv_chunk,
pp_chunk,
pad_chunk,
).to(target_dtype)
).to(self.model_dtype)
encoder_outputs = vt.encoder(
inputs_embeds=inputs_embeds,
attention_mask=~pad_chunk,
@@ -1440,7 +1438,9 @@ class Gemma4ForConditionalGeneration(
frame_valid_lens.append(valid_states.shape[0])
# Project all frames in a single batched call.
flat_valid_states = torch.cat(all_frame_valid_states, dim=0).to(target_dtype)
flat_valid_states = torch.cat(all_frame_valid_states, dim=0).to(
self.model_dtype
)
flat_proj_embs = self.embed_vision(
inputs_embeds=flat_valid_states.unsqueeze(0)
).squeeze(0)
+3 -2
View File
@@ -307,12 +307,13 @@ class Gemma4UnifiedForConditionalGeneration(Gemma4ForConditionalGeneration):
None,
)
if ple_dim is not None and ple_dim > 0:
embed = self.language_model.model.embed_tokens
self.per_layer_embeddings = torch.zeros(
vllm_config.scheduler_config.max_num_batched_tokens,
config.text_config.num_hidden_layers,
ple_dim,
device=self.language_model.model.embed_tokens.weight.device,
dtype=self.language_model.model.embed_tokens.weight.dtype,
device=next(embed.parameters()).device,
dtype=vllm_config.model_config.dtype,
)
else:
self.per_layer_embeddings = None