[Bugfix] Ensure calculated KV scales are applied in attention. (#27232)
Signed-off-by: adabeyta <aabeyta@redhat.com>
This commit is contained in:
@@ -279,6 +279,9 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
# This will be overridden in load_model()
|
||||
self.is_multimodal_pruning_enabled = False
|
||||
self.max_model_len = model_config.max_model_len
|
||||
|
||||
# Always set to false after the first forward pass
|
||||
self.calculate_kv_scales = self.cache_config.calculate_kv_scales
|
||||
self.dcp_world_size = self.parallel_config.decode_context_parallel_size
|
||||
self.dcp_rank = 0 if self.dcp_world_size <= 1 else get_dcp_group().rank_in_group
|
||||
self.max_num_tokens = scheduler_config.max_num_batched_tokens
|
||||
@@ -2625,16 +2628,12 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
)
|
||||
|
||||
# Set cudagraph mode to none if calc_kv_scales is true.
|
||||
if attn_metadata is not None:
|
||||
metadata_list = (
|
||||
attn_metadata.values()
|
||||
if isinstance(attn_metadata, dict)
|
||||
else [attn_metadata]
|
||||
)
|
||||
if any(
|
||||
getattr(m, "enable_kv_scales_calculation", False) for m in metadata_list
|
||||
):
|
||||
cudagraph_runtime_mode = CUDAGraphMode.NONE
|
||||
# KV scales calculation involves dynamic operations that are incompatible
|
||||
# with CUDA graph capture.
|
||||
if self.calculate_kv_scales:
|
||||
cudagraph_runtime_mode = CUDAGraphMode.NONE
|
||||
# Mark KV scales as calculated after the first forward pass
|
||||
self.calculate_kv_scales = False
|
||||
|
||||
# Run the model.
|
||||
# Use persistent buffers for CUDA graphs.
|
||||
|
||||
Reference in New Issue
Block a user