[Bugfix][Quantization] Don't reject fp8_e5m2 KV cache for non-fp8 quantized checkpoints (#45040)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
09f3cd5c10
commit
79ca54d221
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any, cast
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
@@ -166,7 +166,21 @@ def _init_kv_cache_quant(
|
|||||||
# TODO (mgoin): kv cache dtype should be specified in the FP8
|
# TODO (mgoin): kv cache dtype should be specified in the FP8
|
||||||
# checkpoint config and become the "auto" behavior
|
# checkpoint config and become the "auto" behavior
|
||||||
if layer.kv_cache_dtype == "fp8_e5m2":
|
if layer.kv_cache_dtype == "fp8_e5m2":
|
||||||
raise ValueError("fp8_e5m2 kv-cache is not supported with fp8 checkpoints.")
|
# A compressed-tensors checkpoint stores fp8 KV scales only when it
|
||||||
|
# declares a kv_cache_scheme; weight-only ones declare none and must
|
||||||
|
# keep fp8_e5m2, the only fp8 KV dtype usable on Ampere.
|
||||||
|
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||||
|
CompressedTensorsConfig,
|
||||||
|
CompressedTensorsKVCacheMethod,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not isinstance(quant_method, CompressedTensorsKVCacheMethod) or (
|
||||||
|
cast(CompressedTensorsConfig, quant_method.quant_config).kv_cache_scheme
|
||||||
|
is not None
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"fp8_e5m2 kv-cache is not supported with fp8 checkpoints."
|
||||||
|
)
|
||||||
# If quantization is enabled, we make "k_scale" and "v_scale"
|
# If quantization is enabled, we make "k_scale" and "v_scale"
|
||||||
# parameters so that it can be loaded from the model checkpoint.
|
# parameters so that it can be loaded from the model checkpoint.
|
||||||
# The k/v_scale will then be converted back to native float32
|
# The k/v_scale will then be converted back to native float32
|
||||||
|
|||||||
Reference in New Issue
Block a user