Add NVFP4 KV 4-over-6 scale search (#45187)

Signed-off-by: weimingc <17592131+meenchen@users.noreply.github.com>
This commit is contained in:
Wei-Ming Chen
2026-08-07 13:22:27 -07:00
committed by GitHub
parent 70456e5e6f
commit 1c94e8dc7d
11 changed files with 284 additions and 42 deletions
@@ -459,7 +459,8 @@ class Attention(nn.Module, AttentionLayerBase):
if (
self.impl.supports_quant_query_input
and (
self.kv_cache_dtype.startswith("fp8") or self.kv_cache_dtype == "nvfp4"
self.kv_cache_dtype.startswith("fp8")
or self.kv_cache_dtype.startswith("nvfp4")
)
and not self.kv_cache_dtype.endswith("per_token_head")
):
@@ -502,7 +503,9 @@ class Attention(nn.Module, AttentionLayerBase):
# which reduces overheads during decoding.
# Otherwise queries are quantized using custom ops
# which causes decoding overheads
assert self.kv_cache_dtype in {"fp8", "fp8_e4m3", "nvfp4"}
assert self.kv_cache_dtype in {"fp8", "fp8_e4m3"} or (
self.kv_cache_dtype.startswith("nvfp4")
)
# check if query quantization is supported
if self.impl.supports_quant_query_input: