[torch.compile] reorganize the cache directory to support compiling multiple models (#19064)
Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
@@ -320,8 +320,10 @@ class EagleProposer:
|
||||
target_attn_layer_names = set(
|
||||
get_layers_from_vllm_config(self.vllm_config, Attention).keys())
|
||||
|
||||
self.model = get_model(vllm_config=self.vllm_config,
|
||||
model_config=draft_model_config)
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
with set_model_tag("eagle_head"):
|
||||
self.model = get_model(vllm_config=self.vllm_config,
|
||||
model_config=draft_model_config)
|
||||
|
||||
draft_attn_layer_names = (
|
||||
get_layers_from_vllm_config(self.vllm_config, Attention).keys() -
|
||||
|
||||
@@ -48,9 +48,11 @@ class MedusaProposer:
|
||||
return [list(row) for row in zip(*draft_tokens)]
|
||||
|
||||
def load_model(self, target_model: nn.Module) -> None:
|
||||
self.model = get_model(vllm_config=self.vllm_config,
|
||||
model_config=self.vllm_config.
|
||||
speculative_config.draft_model_config)
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
with set_model_tag("medusa_head"):
|
||||
self.model = get_model(vllm_config=self.vllm_config,
|
||||
model_config=self.vllm_config.
|
||||
speculative_config.draft_model_config)
|
||||
|
||||
@torch.inference_mode()
|
||||
def dummy_run(self, num_tokens: int) -> None:
|
||||
|
||||
Reference in New Issue
Block a user