[Fix] Fix FSE compatibility detection for Quark-produced models (#54573)
Signed-off-by: xuebwang-amd <xuebwang@amd.com> Signed-off-by: Felix Marty <Felix.Marty@amd.com> Co-authored-by: xuebwang-amd <xuebwang@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
xuebwang-amd
Andreas Karatzas
parent
514c7314a0
commit
2fe5cef35e
@@ -86,7 +86,7 @@ def resolve_layer_fused_shared_expert(
|
||||
if fse_requested and not is_fused_shared_expert_enabled:
|
||||
logger.warning(
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
|
||||
"cannot be enabled: %s.",
|
||||
"cannot be enabled - skipping for this layer: %s.",
|
||||
fse_reason,
|
||||
)
|
||||
return is_fused_shared_expert_enabled
|
||||
|
||||
@@ -138,15 +138,27 @@ def is_shared_expert_quant_fse_compatible(
|
||||
)
|
||||
|
||||
if isinstance(quant_config, QuarkConfig):
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
should_ignore_layer,
|
||||
)
|
||||
|
||||
# TODO: layer_type_quant_config is not taken into account here.
|
||||
assert "exclude" in quant_config.quant_config
|
||||
assert "global_quant_config" in quant_config.quant_config
|
||||
|
||||
is_compatible = not any(
|
||||
"shared_expert" in str(entry)
|
||||
for entry in quant_config.quant_config["exclude"]
|
||||
exclude = quant_config.quant_config["exclude"]
|
||||
|
||||
# should_ignore_layer raises a rightful ValueError in case different shards
|
||||
# have a different ignore policy, as unsupported.
|
||||
is_excluded = any(
|
||||
should_ignore_layer(
|
||||
f"{shared_expert_prefix}.{projection_name}",
|
||||
ignore=exclude,
|
||||
fused_mapping=quant_config.packed_modules_mapping,
|
||||
)
|
||||
for projection_name in projection_names
|
||||
)
|
||||
if not is_compatible:
|
||||
if is_excluded:
|
||||
return False, f"Quark excludes shared experts at {shared_expert_prefix}"
|
||||
|
||||
global_quant_config = quant_config.quant_config["global_quant_config"]
|
||||
|
||||
Reference in New Issue
Block a user