[Core] Add MRV2 virtual-batch PCP for MLA (#46570)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-07-19 02:53:15 +00:00
committed by GitHub
co-authored by Codex OpenAI Codex Matthew Bonanni
parent 9243e0124e
commit b6ff8a2f50
39 changed files with 1204 additions and 196 deletions
+14 -11
View File
@@ -1784,7 +1784,7 @@ def initialize_model_parallel(
get_world_group().device_group
)
# the layout order is: ExternalDP x DP x PP x TP
# the layout order is: ExternalDP x DP x PP x PCP x TP
# ExternalDP is the data parallel group that is not part of the model,
# every dp rank can generate independently (in verl integration).
# DP is the data parallel group that is part of the model,
@@ -1821,17 +1821,13 @@ def initialize_model_parallel(
# Build the DCP model-parallel groups.
global _DCP
assert _DCP is None, "decode context model parallel group is already initialized"
# Note(hc): In the current implementation of decode context parallel,
# dcp_size must not exceed tp_size, because the world size does not
# change by DCP, it simply reuses the GPUs of TP group, and split one
# TP group into tp_size//dcp_size DCP groups.
group_ranks = all_ranks.reshape(-1, decode_context_model_parallel_size).unbind(0)
dcp_size = decode_context_model_parallel_size or 1
dcp_ranks = local_all_ranks if enable_elastic_ep else all_ranks
if dcp_size > 1:
# DCP spans PCP first, then TP for full TP x PCP groups.
dcp_ranks = dcp_ranks.transpose(-1, -2)
group_ranks = dcp_ranks.reshape(-1, dcp_size).unbind(0)
group_ranks = [x.tolist() for x in group_ranks]
if enable_elastic_ep:
group_ranks = local_all_ranks.reshape(
-1, decode_context_model_parallel_size
).unbind(0)
group_ranks = [x.tolist() for x in group_ranks]
_DCP = init_model_parallel_group(
group_ranks,
get_world_group().local_rank,
@@ -2005,6 +2001,13 @@ def ensure_model_parallel_initialized(
f"{pcp_world_size=} vs. "
f"{prefill_context_model_parallel_size=}"
)
dcp_world_size = get_dcp_group().world_size
dcp_model_parallel_size = decode_context_model_parallel_size or 1
assert dcp_world_size == dcp_model_parallel_size, (
"decode context parallel group already initialized, but of unexpected size: "
f"{dcp_world_size=} vs. "
f"{dcp_model_parallel_size=}"
)
def prepare_communication_buffer_for_model(model: torch.nn.Module):