Co-authored-by: vllm-ci-failure-analyzer <kevin@inferact.ai>
This commit is contained in:
co-authored by
vllm-ci-failure-analyzer
parent
7a2fdbaac4
commit
ba53da60bb
@@ -191,7 +191,6 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routing_method_type: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
tune_max_num_tokens: int = 8192,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
|
||||
@@ -212,7 +211,6 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
topk_group: Top-k within groups (default: None -> 0)
|
||||
e_score_correction_bias: Optional routing bias. dtype: bfloat16
|
||||
routing_method_type: FlashInfer RoutingMethodType enum value
|
||||
tune_max_num_tokens: Maximum token count covered by autotuning.
|
||||
|
||||
Returns:
|
||||
Output tensor from MoE layer. dtype: same as x (bfloat16)
|
||||
@@ -264,7 +262,7 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
enable_pdl=None,
|
||||
do_finalize=True,
|
||||
output=None,
|
||||
tune_max_num_tokens=tune_max_num_tokens,
|
||||
tune_max_num_tokens=8192,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
if isinstance(out, (tuple, list)):
|
||||
|
||||
Reference in New Issue
Block a user