[Bugfix][Qwen4Exp] Load PLE tables unquantized under Quark checkpoints (#59443)
Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
Cursor
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
TJian
parent
c04c79e50d
commit
edca360f13
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.modelopt import (
|
|||||||
ModelOptNvFp4Config,
|
ModelOptNvFp4Config,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
|
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
|
||||||
|
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||||
from vllm.models.qwen4_exp.amd import ple_layer as amd_ple_layer
|
from vllm.models.qwen4_exp.amd import ple_layer as amd_ple_layer
|
||||||
from vllm.models.qwen4_exp.amd.ple_layer import (
|
from vllm.models.qwen4_exp.amd.ple_layer import (
|
||||||
Qwen4ExpPLELayer as Qwen4ExpPLELayerAMD,
|
Qwen4ExpPLELayer as Qwen4ExpPLELayerAMD,
|
||||||
@@ -585,6 +586,16 @@ def test_ple_embedding_respects_inc_layer_config() -> None:
|
|||||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix)
|
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix)
|
||||||
|
|
||||||
|
|
||||||
|
def test_ple_embedding_is_unquantized_under_quark() -> None:
|
||||||
|
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||||
|
quant_config = QuarkConfig({"exclude": [], "global_quant_config": {}})
|
||||||
|
|
||||||
|
assert isinstance(
|
||||||
|
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||||
|
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
|
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
|
||||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||||
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
|
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.quantization.modelopt import (
|
|||||||
ModelOptMixedPrecisionConfig,
|
ModelOptMixedPrecisionConfig,
|
||||||
ModelOptQuantConfigBase,
|
ModelOptQuantConfigBase,
|
||||||
)
|
)
|
||||||
|
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||||
create_fp8_scale_parameter,
|
create_fp8_scale_parameter,
|
||||||
)
|
)
|
||||||
@@ -196,6 +197,9 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
|
|||||||
and not quant_config.config_parser.resolve(None, prefix).quantized
|
and not quant_config.config_parser.resolve(None, prefix).quantized
|
||||||
):
|
):
|
||||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||||
|
# Quark quantizes only Linear and MoE layers; PLE tables stay BF16.
|
||||||
|
if isinstance(quant_config, QuarkConfig):
|
||||||
|
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||||
if not isinstance(quant_config, Fp8Config):
|
if not isinstance(quant_config, Fp8Config):
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"Qwen4Exp PLE embedding does not support quantization config "
|
"Qwen4Exp PLE embedding does not support quantization config "
|
||||||
|
|||||||
Reference in New Issue
Block a user