Add NVFP4 KV 4-over-6 scale search (#45187)
Signed-off-by: weimingc <17592131+meenchen@users.noreply.github.com>
This commit is contained in:
@@ -459,7 +459,8 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
if (
|
||||
self.impl.supports_quant_query_input
|
||||
and (
|
||||
self.kv_cache_dtype.startswith("fp8") or self.kv_cache_dtype == "nvfp4"
|
||||
self.kv_cache_dtype.startswith("fp8")
|
||||
or self.kv_cache_dtype.startswith("nvfp4")
|
||||
)
|
||||
and not self.kv_cache_dtype.endswith("per_token_head")
|
||||
):
|
||||
@@ -502,7 +503,9 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
# which reduces overheads during decoding.
|
||||
# Otherwise queries are quantized using custom ops
|
||||
# which causes decoding overheads
|
||||
assert self.kv_cache_dtype in {"fp8", "fp8_e4m3", "nvfp4"}
|
||||
assert self.kv_cache_dtype in {"fp8", "fp8_e4m3"} or (
|
||||
self.kv_cache_dtype.startswith("nvfp4")
|
||||
)
|
||||
|
||||
# check if query quantization is supported
|
||||
if self.impl.supports_quant_query_input:
|
||||
|
||||
Reference in New Issue
Block a user