Add nvfp4 kv cache support (#40177)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
@@ -114,12 +114,12 @@ QUANT_ALGOS = [
|
||||
# MIXED_PRECISION,
|
||||
"MIXED_PRECISION",
|
||||
]
|
||||
KV_CACHE_QUANT_ALGOS = ["FP8"]
|
||||
KV_CACHE_QUANT_ALGOS = ["FP8", "NVFP4"]
|
||||
|
||||
|
||||
class ModelOptFp8KVCacheMethod(BaseKVCacheMethod):
|
||||
class ModelOptKVCacheMethod(BaseKVCacheMethod):
|
||||
"""
|
||||
Supports loading kv-cache scaling factors from FP8 checkpoints.
|
||||
Supports loading kv-cache scaling factors from FP8 or NVFP4 checkpoints.
|
||||
"""
|
||||
|
||||
def __init__(self, quant_config: "ModelOptQuantConfigBase"):
|
||||
@@ -995,7 +995,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
ModelOptFp8Config.LinearMethodCls = ModelOptFp8LinearMethod
|
||||
ModelOptFp8Config.FusedMoEMethodCls = ModelOptFp8MoEMethod
|
||||
ModelOptFp8Config.KVCacheMethodCls = ModelOptFp8KVCacheMethod
|
||||
ModelOptFp8Config.KVCacheMethodCls = ModelOptKVCacheMethod
|
||||
|
||||
|
||||
class ModelOptNvFp4Config(ModelOptQuantConfigBase):
|
||||
@@ -1488,7 +1488,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
|
||||
ModelOptNvFp4Config.LinearMethodCls = ModelOptNvFp4LinearMethod
|
||||
ModelOptNvFp4Config.FusedMoEMethodCls = ModelOptNvFp4FusedMoE
|
||||
ModelOptNvFp4Config.KVCacheMethodCls = ModelOptFp8KVCacheMethod
|
||||
ModelOptNvFp4Config.KVCacheMethodCls = ModelOptKVCacheMethod
|
||||
|
||||
|
||||
class ModelOptMxFp8Config(ModelOptQuantConfigBase):
|
||||
@@ -2018,7 +2018,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
# Register the method classes for ModelOptMxFp8Config
|
||||
ModelOptMxFp8Config.LinearMethodCls = ModelOptMxFp8LinearMethod
|
||||
ModelOptMxFp8Config.FusedMoEMethodCls = ModelOptMxFp8FusedMoE
|
||||
ModelOptMxFp8Config.KVCacheMethodCls = ModelOptFp8KVCacheMethod
|
||||
ModelOptMxFp8Config.KVCacheMethodCls = ModelOptKVCacheMethod
|
||||
|
||||
|
||||
class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
@@ -2166,7 +2166,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
# KV-cache quantization
|
||||
if isinstance(layer, Attention):
|
||||
if self.kv_cache_quant_method:
|
||||
return ModelOptFp8KVCacheMethod(self)
|
||||
return ModelOptKVCacheMethod(self)
|
||||
return None
|
||||
|
||||
# Excluded layers
|
||||
|
||||
Reference in New Issue
Block a user