[Quant] Support compressed-tensors WNA8O8Int linears and WNInt embeddings (#44340)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-06-04 07:40:33 -07:00
committed by GitHub
parent b5235fca2e
commit 06ee2d8433
14 changed files with 744 additions and 27 deletions
+3 -3
View File
@@ -1057,7 +1057,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
"normalizer",
torch.tensor(
config.hidden_size**0.5,
dtype=self.embed_tokens.weight.dtype,
dtype=vllm_config.model_config.dtype,
),
persistent=False,
)
@@ -1111,7 +1111,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
)
self.hidden_states = torch.zeros(
(max_num_tokens, config.hidden_size),
dtype=self.embed_tokens.weight.dtype,
dtype=vllm_config.model_config.dtype,
device=device,
)
if (
@@ -1124,7 +1124,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
config.num_hidden_layers,
self.hidden_size_per_layer_input,
),
dtype=self.embed_tokens.weight.dtype,
dtype=vllm_config.model_config.dtype,
device=device,
)
else: