Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -14,7 +14,6 @@ from tqdm import tqdm
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_dp_group,
|
||||
get_ep_group,
|
||||
is_global_first_rank,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
@@ -356,7 +355,6 @@ def _get_fp8_fp4_mega_moe_m_values(
|
||||
dg = _import_deep_gemm()
|
||||
gate, experts = module.gate, module.experts
|
||||
uses_mega_gate = _mega_moe_uses_mega_gate(module)
|
||||
num_max_tokens_per_rank = experts.get_symm_buffer().num_max_tokens_per_rank
|
||||
if module.use_sequence_parallel:
|
||||
max_tokens = cdiv(max_tokens, module.tp_size)
|
||||
|
||||
@@ -369,14 +367,7 @@ def _get_fp8_fp4_mega_moe_m_values(
|
||||
if uses_mega_gate and hasattr(dg, "get_bf16_mega_gate_config")
|
||||
else {}
|
||||
)
|
||||
block_m = dg.get_block_m_for_mega_moe(
|
||||
get_ep_group().world_size,
|
||||
experts.num_experts,
|
||||
num_max_tokens_per_rank,
|
||||
num_tokens,
|
||||
experts.top_k,
|
||||
"fp8xfp4",
|
||||
)
|
||||
block_m = experts.get_block_m(num_tokens)
|
||||
# Keep the largest M: DeepSeek-V4 only calls bf16_mega_gate above a
|
||||
# small token threshold, and a config's largest M is always above it.
|
||||
m_values[(tuple(gate_config.items()), block_m)] = num_tokens
|
||||
|
||||
@@ -30,6 +30,7 @@ _DEEPSEEK_V4_SPARSE_MLA_BACKENDS = frozenset(
|
||||
"ROCM_FLASHMLA_SPARSE_DSV4",
|
||||
"FLASHMLA_SPARSE_DSV41",
|
||||
"FLASHINFER_MLA_SPARSE_DSV41",
|
||||
"ROCM_FLASHMLA_SPARSE_DSV41",
|
||||
"DEEPSEEK_SPARSE_SWA",
|
||||
}
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user