[Online quantization] Add targeted online quantization configuration based on user patterns (#51285)

This commit is contained in:
fxmarty-amd
2026-09-02 08:55:15 -04:00
committed by GitHub
parent 2a4e3cc3db
commit 7894394b07
11 changed files with 892 additions and 172 deletions
@@ -5,9 +5,12 @@ from collections.abc import Iterable, Mapping
from types import MappingProxyType
from typing import Any
import regex as re
import torch
from vllm.model_executor.layers.quantization.utils.config_utils import (
find_matching_patterns,
)
QuarkQTensorHint = dict[str, Any] | list[dict[str, Any]] | None
@@ -49,80 +52,18 @@ def should_ignore_layer(
):
return True
# layer_name = model.layers.0.self_attn.qkv_proj
# proj_name = qkv_proj
proj_name = layer_name.split(".")[-1]
# Fused layers like gate_up_proj or qkv_proj will not be fused
# in the safetensors checkpoint. So, we convert the name
# from the fused version to unfused + check to make sure that
# each shard of the fused layer has the same scheme.
if proj_name in fused_mapping:
shard_proj_names = fused_mapping[proj_name]
# Convert fused_name --> [shard_names]
shard_names = [
layer_name.replace(proj_name, shard_proj_name)
for shard_proj_name in shard_proj_names
]
# Layer should be ignored if shards are ignored.
should_ignore_layer = None
for shard_name in shard_names:
should_ignore_shard = check_equal_or_regex_match(
layer_name=shard_name, targets=ignore
)
# If shard_idx=0, set layer ignore to match shard.
if should_ignore_layer is None:
should_ignore_layer = should_ignore_shard
# If shard_idx=1+ confirm scheme matches prior shards.
elif should_ignore_shard != should_ignore_layer:
raise ValueError(
f"Found a different quantization schemes for "
f"{shard_proj_names} in {layer_name}. vLLM "
"requires all to use the same scheme."
)
# Unfused layers like down_proj and o_proj will match
# the safetensors checkpoint already.
else:
should_ignore_layer = check_equal_or_regex_match(
layer_name=layer_name, targets=ignore
# A direct fused-layer pattern takes precedence over expansion. For
# model.layers.0.self_attn.qkv_proj,
# ignore=["re:.*qkv_proj.*"] yields [{"re:.*qkv_proj.*"}]. In contrast,
# ignore=["re:.*[qkv]_proj"] yields one matching set per expanded shard.
per_shard_matches = find_matching_patterns(layer_name, ignore, fused_mapping)
shards_ignored = [len(matches) > 0 for matches in per_shard_matches]
if any(shards_ignored) and not all(shards_ignored):
raise ValueError(
f"Found different quantization schemes for the shards of "
f"{layer_name}. vLLM requires all to use the same scheme."
)
assert should_ignore_layer is not None
return should_ignore_layer
def check_equal_or_regex_match(layer_name: str, targets: Iterable[str]) -> bool:
"""
Checks whether a layer_name is exactly equal or a regex match for
if target starts with 're:' to any target in list.
"""
return any(_is_equal_or_regex_match(layer_name, target) for target in targets)
def _is_equal_or_regex_match(
value: str, target: str, check_contains: bool = False
) -> bool:
"""
Checks whether a value is exactly equal or a regex match for target
if target starts with 're:'. If check_contains is set to True,
additionally checks if the target string is contained within the value.
"""
if target.startswith("re:"):
pattern = target[3:]
if re.match(pattern, value):
return True
elif check_contains:
if target.lower() in value.lower():
return True
elif target == value:
return True
return False
return all(shards_ignored)
# utility for tensor dims > 2 cases