[Bugfix][Model] Fix FP8 PLE loading in mixed ModelOpt checkpoints (#54882)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
This commit is contained in:
co-authored by
Vadim Gimpelson
parent
d410fc12f3
commit
d4d703caf9
@@ -22,6 +22,9 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptMixedPrecisionConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
create_fp8_scale_parameter,
|
||||
create_fp8_weight_parameter,
|
||||
@@ -133,6 +136,11 @@ def _get_ple_embedding_quant_method(
|
||||
) -> QuantizeMethodBase | None:
|
||||
"""Select global-scale FP8 only for quantized PLE checkpoint shards."""
|
||||
|
||||
if isinstance(quant_config, ModelOptMixedPrecisionConfig):
|
||||
if quant_config._resolve_quant_algo(prefix) == "FP8":
|
||||
return Qwen4ExpPLEFp8EmbeddingMethod()
|
||||
return None
|
||||
|
||||
if not isinstance(quant_config, Fp8Config):
|
||||
return None
|
||||
if not quant_config.is_checkpoint_fp8_serialized:
|
||||
|
||||
Reference in New Issue
Block a user