[Bugfix][HiSparse] Fix MTP acceptance collapse under FULL graphs with a saturated GPU pool (#59309)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Matthew Bonanni
parent
0cbac6cd13
commit
4056c8ac1f
@@ -224,13 +224,13 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
self.model = self.load_draft_model(target_model, target_attn_layer_names)
|
||||
self._validate_local_argmax_reduction()
|
||||
|
||||
all_attn_layers = set[str](
|
||||
get_layers_from_vllm_config(
|
||||
self.vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
).keys()
|
||||
all_attn_layers = get_layers_from_vllm_config(
|
||||
self.vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
)
|
||||
self.draft_attn_layer_names = all_attn_layers - target_attn_layer_names
|
||||
self.draft_attn_layer_names = set(all_attn_layers) - target_attn_layer_names
|
||||
for layer_name in self.draft_attn_layer_names:
|
||||
all_attn_layers[layer_name].is_draft_layer = True
|
||||
|
||||
target_supports_mm = self.vllm_config.model_config.supports_multimodal_inputs
|
||||
draft_supports_mm = supports_multimodal_embeddings(self.model)
|
||||
|
||||
Reference in New Issue
Block a user