[ModelOpt] Redesign the LinearMethod classes using the generic QuantKey-driven method (#49381)
Signed-off-by: Juhi Mittal <juhim@nvidia.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
mgoin
parent
4bf06be985
commit
7a977c0699
@@ -377,12 +377,18 @@ def _get_kv_b_proj_input_dtype(
|
||||
return None
|
||||
if weight_dtype == current_platform.fp8_dtype():
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptFp8PbWoLinearMethod,
|
||||
ModelOptLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Static128BlockSym,
|
||||
)
|
||||
|
||||
quant_method = getattr(kv_b_proj, "quant_method", None)
|
||||
# FP8_PB_WO dynamically quantizes BF16/FP16 inputs in the linear method.
|
||||
if isinstance(quant_method, ModelOptFp8PbWoLinearMethod):
|
||||
if (
|
||||
isinstance(quant_method, ModelOptLinearMethod)
|
||||
and quant_method.spec.weight is kFp8Static128BlockSym
|
||||
):
|
||||
return quant_method.input_dtype
|
||||
if not use_fp8_prefill:
|
||||
return None
|
||||
|
||||
@@ -55,13 +55,10 @@ WEIGHT_LOADER_V2_SUPPORTED = [
|
||||
"AutoGPTQLinearMethod",
|
||||
"Fp8LinearMethod",
|
||||
"FBGEMMFp8LinearMethod",
|
||||
"ModelOptFp8LinearMethod",
|
||||
"ModelOptFp8PcPtLinearMethod",
|
||||
"ModelOptFp8PbWoLinearMethod",
|
||||
"QuarkLinearMethod",
|
||||
"ModelOptNvFp4LinearMethod",
|
||||
"ModelOptNvFp4W4A16LinearMethod",
|
||||
"HummingLinearMethod",
|
||||
# ModelOptLinearMethod self-registers via
|
||||
# register_weight_loader_v2_supported_method (see modelopt.py).
|
||||
]
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -38,6 +38,11 @@ from vllm.utils.platform_utils import get_device_name_as_file_name
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Pre-fill value for scale parameters whose shards load independently. The
|
||||
# shards are combined with .max(), so an unloaded shard must never win; the
|
||||
# smallest representable float32 guarantees that.
|
||||
FP8_SCALE_SENTINEL = torch.finfo(torch.float32).min
|
||||
|
||||
|
||||
def is_fp8(x: torch.dtype | torch.Tensor) -> bool:
|
||||
if isinstance(x, torch.Tensor):
|
||||
|
||||
Reference in New Issue
Block a user