[Bugfix][MoE] Snapshot max_cudagraph_capture_size into FusedMoEConfig (#44613)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
eafbb06331
commit
f87df1df9e
@@ -1266,6 +1266,8 @@ class FusedMoEConfig:
|
|||||||
# cannot silently select one and drop the clamp.
|
# cannot silently select one and drop the clamp.
|
||||||
swiglu_limit: float | None = None
|
swiglu_limit: float | None = None
|
||||||
|
|
||||||
|
max_capture_size: int = 0
|
||||||
|
|
||||||
def __post_init__(self):
|
def __post_init__(self):
|
||||||
if self.dp_size > 1:
|
if self.dp_size > 1:
|
||||||
logger.debug_once(
|
logger.debug_once(
|
||||||
|
|||||||
@@ -4,7 +4,6 @@
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||||
from vllm.config import get_current_vllm_config
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||||
from vllm.model_executor.layers.fused_moe.config import (
|
from vllm.model_executor.layers.fused_moe.config import (
|
||||||
@@ -94,9 +93,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
|||||||
# - pass per-block weight scales to the kernel
|
# - pass per-block weight scales to the kernel
|
||||||
# - skip input activation quantization (kernel applies scaling)
|
# - skip input activation quantization (kernel applies scaling)
|
||||||
self.use_deepseek_fp8_block_scale = quant_config.is_block_quantized
|
self.use_deepseek_fp8_block_scale = quant_config.is_block_quantized
|
||||||
self.max_capture_size = (
|
self.max_capture_size = moe_config.max_capture_size
|
||||||
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
|
|
||||||
)
|
|
||||||
self.gemm1_clamp_limit: torch.Tensor | None = None
|
self.gemm1_clamp_limit: torch.Tensor | None = None
|
||||||
if quant_config.gemm1_clamp_limit is not None:
|
if quant_config.gemm1_clamp_limit is not None:
|
||||||
self.gemm1_clamp_limit = torch.tensor(
|
self.gemm1_clamp_limit = torch.tensor(
|
||||||
|
|||||||
@@ -79,11 +79,7 @@ class TrtLlmMxfp4ExpertsBase:
|
|||||||
else:
|
else:
|
||||||
self.gemm1_clamp_limit = None
|
self.gemm1_clamp_limit = None
|
||||||
|
|
||||||
from vllm.config import get_current_vllm_config
|
self.max_capture_size = moe_config.max_capture_size
|
||||||
|
|
||||||
self.max_capture_size = (
|
|
||||||
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _supports_current_device() -> bool:
|
def _supports_current_device() -> bool:
|
||||||
|
|||||||
@@ -326,6 +326,7 @@ class FusedMoE(PluggableLayer):
|
|||||||
moe_backend=vllm_config.kernel_config.moe_backend,
|
moe_backend=vllm_config.kernel_config.moe_backend,
|
||||||
router_logits_dtype=router_logits_dtype,
|
router_logits_dtype=router_logits_dtype,
|
||||||
max_num_tokens=max_num_batched_tokens,
|
max_num_tokens=max_num_batched_tokens,
|
||||||
|
max_capture_size=compilation_config.max_cudagraph_capture_size,
|
||||||
has_bias=has_bias,
|
has_bias=has_bias,
|
||||||
is_act_and_mul=is_act_and_mul,
|
is_act_and_mul=is_act_and_mul,
|
||||||
is_lora_enabled=vllm_config.lora_config is not None,
|
is_lora_enabled=vllm_config.lora_config is not None,
|
||||||
|
|||||||
@@ -3,7 +3,6 @@
|
|||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm.config import get_current_vllm_config
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.attention import Attention
|
from vllm.model_executor.layers.attention import Attention
|
||||||
from vllm.model_executor.layers.fused_moe import (
|
from vllm.model_executor.layers.fused_moe import (
|
||||||
@@ -141,9 +140,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
|||||||
self.weight_dtype = "gpt_oss_mxfp4"
|
self.weight_dtype = "gpt_oss_mxfp4"
|
||||||
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
|
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
|
||||||
|
|
||||||
self.max_capture_size = (
|
self.max_capture_size = moe.max_capture_size
|
||||||
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
|
|
||||||
)
|
|
||||||
|
|
||||||
self._cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
|
self._cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
|
||||||
self.moe_kernel: mk.FusedMoEKernel | None = None
|
self.moe_kernel: mk.FusedMoEKernel | None = None
|
||||||
@@ -475,9 +472,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
|||||||
self.weight_dtype = "mxfp4"
|
self.weight_dtype = "mxfp4"
|
||||||
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe)
|
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe)
|
||||||
|
|
||||||
self.max_capture_size = (
|
self.max_capture_size = moe.max_capture_size
|
||||||
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
|
|
||||||
)
|
|
||||||
|
|
||||||
self._cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
|
self._cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
|
||||||
self.moe_kernel: mk.FusedMoEKernel | None = None
|
self.moe_kernel: mk.FusedMoEKernel | None = None
|
||||||
|
|||||||
Reference in New Issue
Block a user