[Bugfix][DSv4.1] Match decoder-replay input_ids dtype for FlashInfer MegaMoE (#61041)
Signed-off-by: Shang Wang <samshang.wang@mail.utoronto.ca> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f3cb10b690
commit
ffec221276
@@ -10,7 +10,7 @@ import torch
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.config.kernel import MEGA_MOE_BACKENDS
|
||||
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.forward_context import get_forward_context, override_forward_context
|
||||
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
@@ -56,8 +56,9 @@ class DecoderReplayCudaGraphManager(CudaGraphManager):
|
||||
config, act = vllm_config.model_config.hf_config, vllm_config.model_config.dtype
|
||||
assert isinstance(act, torch.dtype)
|
||||
hc, h, f32 = config.hc_mult, config.hidden_size, torch.float32
|
||||
mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
|
||||
ids = torch.int64 if mega_moe else torch.int32
|
||||
# DeepseekV4Model.forward casts input_ids to int64 for native MegaMoE only.
|
||||
native = vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
|
||||
ids = torch.int64 if native else torch.int32
|
||||
# hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual
|
||||
shapes = [(h,), (), (), (hc,), (hc, 1), (hc, hc), (hc, h)]
|
||||
dtypes = [act, torch.int64, ids, f32, f32, f32, act]
|
||||
|
||||
Reference in New Issue
Block a user