[torch.compile] support all attention backends (#10558)

Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
youkaichao
2024-11-22 14:04:42 -08:00
committed by GitHub
parent db100c5cde
commit eebad39f26
77 changed files with 876 additions and 648 deletions
+2 -1
View File
@@ -273,7 +273,8 @@ class TP1DraftModelRunner(ModelRunner):
if previous_hidden_states is not None else {}
# Run model
with set_forward_context(model_input.attn_metadata):
with set_forward_context(model_input.attn_metadata,
self.vllm_config):
hidden_states = model_executable(
input_ids=model_input.input_tokens,
positions=model_input.input_positions,