[BugFix][torch.compile] KV scale calculation issues with FP8 quantization (#25513)

Signed-off-by: adabeyta <aabeyta@redhat.com>
This commit is contained in:
Adrian Abeyta
2025-09-29 15:52:04 -04:00
committed by GitHub
parent d5ab28511c
commit c42ff4f4fd
3 changed files with 64 additions and 3 deletions
+9
View File
@@ -2351,6 +2351,15 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
self.cudagraph_dispatcher.dispatch(batch_descriptor,
use_cascade_attn)
# Set cudagraph mode to none if calc_kv_scales is true.
if attn_metadata is not None:
metadata_list = (attn_metadata.values() if isinstance(
attn_metadata, dict) else [attn_metadata])
if any(
getattr(m, 'enable_kv_scales_calculation', False)
for m in metadata_list):
cudagraph_runtime_mode = CUDAGraphMode.NONE
# This is currently to get around the assert in the DPMetadata
# where it wants `num_tokens_across_dp` to align with `num_tokens`
if ubatch_slices is not None: