[Core] NGram GPU Implementation compatible with Async Scheduler (#29184)
This commit is contained in:
@@ -907,6 +907,13 @@ class VllmBackend:
|
||||
# Honors opt-outs such as CompilationMode.NONE or VLLM_DISABLE_COMPILE_CACHE.
|
||||
disable_cache = not is_compile_cache_enabled(self.inductor_config)
|
||||
|
||||
# TODO(patchy): ngram gpu kernel will cause vllm torch compile cache errors.
|
||||
is_ngram_gpu_enabled = (
|
||||
vllm_config.speculative_config is not None
|
||||
and vllm_config.speculative_config.use_ngram_gpu()
|
||||
)
|
||||
disable_cache = disable_cache or is_ngram_gpu_enabled
|
||||
|
||||
if disable_cache:
|
||||
logger.info_once("vLLM's torch.compile cache is disabled.", scope="local")
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user