[platform] support custom torch.compile backend key (#11318)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
Signed-off-by: youkaichao <youkaichao@gmail.com>
Co-authored-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
wangxiyuan
2025-01-10 23:46:51 +08:00
committed by GitHub
co-authored by youkaichao
parent 12664ddda5
commit 20410b2fda
5 changed files with 14 additions and 5 deletions
@@ -9,6 +9,7 @@ import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.layers.spec_decode_base_sampler import (
SpecDecodeStochasticBaseSampler)
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -368,7 +369,7 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
# Note that we always sample with replacement.
# probs will be modified in place, but this is fine, as we pass
# in a copy already.
@torch.compile(dynamic=True)
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
def _multinomial(
probs: torch.Tensor,
num_samples: int,