[Compressed-Tensors] Support Kimi-K3 quantized models (#50500)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
This commit is contained in:
@@ -951,7 +951,13 @@ class KimiDecoderLayer(nn.Module):
|
|||||||
return hidden_states, prefix_sum, residual
|
return hidden_states, prefix_sum, residual
|
||||||
|
|
||||||
|
|
||||||
class KimiLinearModel(nn.Module, EagleModelMixin):
|
class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
|
||||||
|
packed_modules_mapping = {
|
||||||
|
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||||
|
"in_proj_qkvgfab": ["q_proj", "k_proj", "v_proj", "b_proj", "f_a_proj"],
|
||||||
|
"conv1d": ["q_conv1d", "k_conv1d", "v_conv1d"],
|
||||||
|
}
|
||||||
|
|
||||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
@@ -1221,6 +1227,7 @@ class KimiLinearModel(nn.Module, EagleModelMixin):
|
|||||||
else:
|
else:
|
||||||
expert_params_mapping = []
|
expert_params_mapping = []
|
||||||
params_dict = dict(self.named_parameters())
|
params_dict = dict(self.named_parameters())
|
||||||
|
|
||||||
# Under the MXFP4 quant interface the routed experts register unpacked
|
# Under the MXFP4 quant interface the routed experts register unpacked
|
||||||
# params (``w13_weight``), while the compressed-tensors checkpoint names
|
# params (``w13_weight``), while the compressed-tensors checkpoint names
|
||||||
# them ``.weight_packed``. Rebind so the expert mapping resolves; scales
|
# them ``.weight_packed``. Rebind so the expert mapping resolves; scales
|
||||||
|
|||||||
Reference in New Issue
Block a user