[Bugfix] Fix kv_cache_dtype=fp8 without scales for FP8 checkpoints (#6761)

This commit is contained in:
Michael Goin
2024-07-25 09:46:15 -07:00
committed by GitHub
parent 889da130e7
commit 65b1f121c8
2 changed files with 12 additions and 6 deletions
@@ -46,10 +46,8 @@ class BaseKVCacheMethod(QuantizeMethodBase):
elif layer.k_scale < 0.0 and layer.v_scale < 0.0:
# If no scales were loaded (both scales are invalid negative
# values), use the default value of 1.0
k_scale = torch.nn.Parameter(torch.tensor(1.0),
requires_grad=False)
v_scale = torch.nn.Parameter(torch.tensor(1.0),
requires_grad=False)
k_scale = 1.0
v_scale = 1.0
else:
# If we find a single kv_scale in the checkpoint, we remap
# kv_scale to k_scale during weight loading, and duplicate