From 3194039c0ee82685af434c9f8023304b4a45124b Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Sat, 1 Feb 2025 11:16:19 -0500 Subject: [PATCH] Apply torch.compile to fused_moe/grouped_topk (#12637) --- vllm/model_executor/layers/fused_moe/fused_moe.py | 1 + vllm/model_executor/models/deepseek_v3.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index c966be99ed..c80e6bf074 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -759,6 +759,7 @@ def fused_topk( # This is used by the Deepseek-V2 and Deepseek-V3 model +@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend) def grouped_topk(hidden_states: torch.Tensor, gating_output: torch.Tensor, topk: int, diff --git a/vllm/model_executor/models/deepseek_v3.py b/vllm/model_executor/models/deepseek_v3.py index f6ab53c85f..06ea3dab93 100644 --- a/vllm/model_executor/models/deepseek_v3.py +++ b/vllm/model_executor/models/deepseek_v3.py @@ -27,6 +27,7 @@ from torch import nn from transformers import PretrainedConfig from vllm.attention import Attention, AttentionMetadata +from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, VllmConfig from vllm.distributed import (get_pp_group, get_tensor_model_parallel_world_size, @@ -566,8 +567,7 @@ class DeepseekV3DecoderLayer(nn.Module): return hidden_states, residual -# TODO(simon): check whether we support torch compile for Deepseek V3 -# @support_torch_compile +@support_torch_compile class DeepseekV3Model(nn.Module): fall_back_to_pt_during_load = False