[Quant] Support compressed-tensors WNA8O8Int linears and WNInt embeddings (#44340)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -1057,7 +1057,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
"normalizer",
|
||||
torch.tensor(
|
||||
config.hidden_size**0.5,
|
||||
dtype=self.embed_tokens.weight.dtype,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
),
|
||||
persistent=False,
|
||||
)
|
||||
@@ -1111,7 +1111,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
)
|
||||
self.hidden_states = torch.zeros(
|
||||
(max_num_tokens, config.hidden_size),
|
||||
dtype=self.embed_tokens.weight.dtype,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
device=device,
|
||||
)
|
||||
if (
|
||||
@@ -1124,7 +1124,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
config.num_hidden_layers,
|
||||
self.hidden_size_per_layer_input,
|
||||
),
|
||||
dtype=self.embed_tokens.weight.dtype,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
device=device,
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -1011,14 +1011,17 @@ class Gemma4ForConditionalGeneration(
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.multimodal_config = multimodal_config
|
||||
self.model_dtype = vllm_config.model_config.dtype
|
||||
|
||||
# Only quantize towers when the quant method supports their
|
||||
# dimensions. BNB/torchao handle arbitrary sizes; other methods
|
||||
# (Marlin, FP8, …) require dimensions divisible by 64, which
|
||||
# the vision tower (intermediate_size=4304) does not satisfy.
|
||||
# TODO(mgoin): remove this by fixing kernel padding.
|
||||
if quant_config and quant_config.get_name() in [
|
||||
"bitsandbytes",
|
||||
"torchao",
|
||||
"compressed-tensors",
|
||||
]:
|
||||
tower_quant = quant_config
|
||||
else:
|
||||
@@ -1081,12 +1084,13 @@ class Gemma4ForConditionalGeneration(
|
||||
# Some variants have hidden_size_per_layer_input=None (no PLE).
|
||||
ple_dim = config.text_config.hidden_size_per_layer_input
|
||||
if ple_dim is not None and ple_dim > 0:
|
||||
embed = self.language_model.model.embed_tokens
|
||||
self.per_layer_embeddings = torch.zeros(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.text_config.num_hidden_layers,
|
||||
ple_dim,
|
||||
device=self.language_model.model.embed_tokens.weight.device,
|
||||
dtype=self.language_model.model.embed_tokens.weight.dtype,
|
||||
device=next(embed.parameters()).device,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
)
|
||||
else:
|
||||
self.per_layer_embeddings = None
|
||||
@@ -1246,7 +1250,6 @@ class Gemma4ForConditionalGeneration(
|
||||
vt = self.vision_tower
|
||||
vision_cfg = self.config.vision_config
|
||||
pooling_k2 = vision_cfg.pooling_kernel_size**2
|
||||
target_dtype = self.language_model.model.embed_tokens.weight.dtype
|
||||
|
||||
# Concurrent requests with different image resolutions may
|
||||
# arrive as a list of per-image tensors, while same-resolution
|
||||
@@ -1291,7 +1294,7 @@ class Gemma4ForConditionalGeneration(
|
||||
pv_tensor,
|
||||
pp_tensor,
|
||||
pad_tensor,
|
||||
).to(target_dtype)
|
||||
).to(self.model_dtype)
|
||||
encoder_outputs = vt.encoder(
|
||||
inputs_embeds=inputs_embeds,
|
||||
attention_mask=~pad_tensor,
|
||||
@@ -1328,12 +1331,8 @@ class Gemma4ForConditionalGeneration(
|
||||
all_valid_states[orig_idx] = valid_states
|
||||
valid_lens[orig_idx] = valid_states.shape[0]
|
||||
|
||||
# Use embed_tokens dtype as compute dtype; embedding_projection.weight
|
||||
# may be uint8 under BnB 4-bit, which would corrupt the cast.
|
||||
target_dtype = self.language_model.model.embed_tokens.weight.dtype
|
||||
|
||||
# Project all images in a single batched call.
|
||||
flat_valid_states = torch.cat(all_valid_states, dim=0).to(target_dtype)
|
||||
flat_valid_states = torch.cat(all_valid_states, dim=0).to(self.model_dtype)
|
||||
flat_proj_embs = self.embed_vision(
|
||||
inputs_embeds=flat_valid_states.unsqueeze(0)
|
||||
).squeeze(0)
|
||||
@@ -1373,7 +1372,6 @@ class Gemma4ForConditionalGeneration(
|
||||
vt = self.vision_tower
|
||||
vision_cfg = self.config.vision_config
|
||||
pooling_k2 = vision_cfg.pooling_kernel_size**2
|
||||
target_dtype = self.language_model.model.embed_tokens.weight.dtype
|
||||
|
||||
if isinstance(frame_counts, torch.Tensor):
|
||||
fc_list = frame_counts.tolist()
|
||||
@@ -1405,7 +1403,7 @@ class Gemma4ForConditionalGeneration(
|
||||
pv_chunk,
|
||||
pp_chunk,
|
||||
pad_chunk,
|
||||
).to(target_dtype)
|
||||
).to(self.model_dtype)
|
||||
encoder_outputs = vt.encoder(
|
||||
inputs_embeds=inputs_embeds,
|
||||
attention_mask=~pad_chunk,
|
||||
@@ -1440,7 +1438,9 @@ class Gemma4ForConditionalGeneration(
|
||||
frame_valid_lens.append(valid_states.shape[0])
|
||||
|
||||
# Project all frames in a single batched call.
|
||||
flat_valid_states = torch.cat(all_frame_valid_states, dim=0).to(target_dtype)
|
||||
flat_valid_states = torch.cat(all_frame_valid_states, dim=0).to(
|
||||
self.model_dtype
|
||||
)
|
||||
flat_proj_embs = self.embed_vision(
|
||||
inputs_embeds=flat_valid_states.unsqueeze(0)
|
||||
).squeeze(0)
|
||||
|
||||
@@ -307,12 +307,13 @@ class Gemma4UnifiedForConditionalGeneration(Gemma4ForConditionalGeneration):
|
||||
None,
|
||||
)
|
||||
if ple_dim is not None and ple_dim > 0:
|
||||
embed = self.language_model.model.embed_tokens
|
||||
self.per_layer_embeddings = torch.zeros(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.text_config.num_hidden_layers,
|
||||
ple_dim,
|
||||
device=self.language_model.model.embed_tokens.weight.device,
|
||||
dtype=self.language_model.model.embed_tokens.weight.dtype,
|
||||
device=next(embed.parameters()).device,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
)
|
||||
else:
|
||||
self.per_layer_embeddings = None
|
||||
|
||||
Reference in New Issue
Block a user