Support kimi k3 nvfp4 checkpoint (#53132)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
@@ -36,7 +36,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
sharded_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.parameter import BasevLLMParameter
|
||||
from vllm.model_executor.parameter import BasevLLMParameter, BlockQuantScaleParameter
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.kimi_k3.nvidia.kda_metadata import (
|
||||
KimiK3KDAAttentionBackend,
|
||||
@@ -102,7 +102,11 @@ class _KimiGDNMergedColumnParallelLinear(MergedColumnParallelLinear):
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id == self.replicated_shard_id:
|
||||
replicate_block_scale = (
|
||||
isinstance(param, BlockQuantScaleParameter)
|
||||
and loaded_weight.shape[param.output_dim] < self.tp_size
|
||||
)
|
||||
if loaded_shard_id == self.replicated_shard_id or replicate_block_scale:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
@@ -121,7 +125,11 @@ class _KimiGDNMergedColumnParallelLinear(MergedColumnParallelLinear):
|
||||
) -> None:
|
||||
tp_rank = self.tp_rank
|
||||
param_tp_rank = getattr(param, "tp_rank", None)
|
||||
if loaded_shard_id == self.replicated_shard_id:
|
||||
replicate_block_scale = (
|
||||
isinstance(param, BlockQuantScaleParameter)
|
||||
and loaded_weight.shape[param.output_dim] < self.tp_size
|
||||
)
|
||||
if loaded_shard_id == self.replicated_shard_id or replicate_block_scale:
|
||||
self.tp_rank = 0
|
||||
if param_tp_rank is not None:
|
||||
param.tp_rank = 0
|
||||
|
||||
Reference in New Issue
Block a user