[torch.compile] support all attention backends (#10558)
Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
@@ -273,7 +273,8 @@ class TP1DraftModelRunner(ModelRunner):
|
||||
if previous_hidden_states is not None else {}
|
||||
|
||||
# Run model
|
||||
with set_forward_context(model_input.attn_metadata):
|
||||
with set_forward_context(model_input.attn_metadata,
|
||||
self.vllm_config):
|
||||
hidden_states = model_executable(
|
||||
input_ids=model_input.input_tokens,
|
||||
positions=model_input.input_positions,
|
||||
|
||||
Reference in New Issue
Block a user