[Online quantization] Add targeted online quantization configuration based on user patterns (#51285)
This commit is contained in:
@@ -50,7 +50,10 @@ from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E
|
||||
QuarkW8A8Int8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
QuarkQTensorHint,
|
||||
should_ignore_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
@@ -1294,6 +1297,23 @@ FUSED_MAPPING = {
|
||||
}
|
||||
|
||||
|
||||
def test_quark_should_ignore_layer_checks_children():
|
||||
assert should_ignore_layer(
|
||||
"model.layers.78.mlp.experts",
|
||||
["model.layers.78.mlp.experts.0.down_proj"],
|
||||
check_children=True,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_should_ignore_layer_rejects_partial_fused_matches():
|
||||
with pytest.raises(ValueError, match="different quantization schemes"):
|
||||
should_ignore_layer(
|
||||
"model.layers.0.self_attn.qkv_proj",
|
||||
["model.layers.0.self_attn.q_proj"],
|
||||
FUSED_MAPPING,
|
||||
)
|
||||
|
||||
|
||||
def test_fused_name_listed_directly_is_skipped():
|
||||
# Regression for Step-3.5-Flash-FP8: the checkpoint lists the fused
|
||||
# name (``qkv_proj``) directly in ``modules_to_not_convert``. When a
|
||||
|
||||
Reference in New Issue
Block a user