[Bugfix][PCP] Align MoE routing padding with gathered tokens (#60946)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: wangyicong <wangyicong@bytedance.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Yicong Wang
2026-10-14 21:04:00 -04:00
committed by GitHub
co-authored by Lucas Wilkinson Claude Opus 5.5 OpenAI Codex
parent 8530aed2b7
commit bfdbfa1632
7 changed files with 180 additions and 26 deletions
+5 -2
View File
@@ -743,6 +743,9 @@ class ModelCudaGraphManager(CudaGraphManager):
# Capture with dummy rows marked as padding.
input_buffers.is_padding.fill_(True)
is_padding = input_buffers.is_padding[:num_tokens]
if pcp_manager is not None:
is_padding = pcp_manager.moe_is_padding(is_padding)
def forward_fn(cg_mode: CUDAGraphMode) -> None:
batch_descriptor = None
@@ -760,7 +763,7 @@ class ModelCudaGraphManager(CudaGraphManager):
num_tokens_across_dp=num_tokens_across_dp,
slot_mapping=slot_mappings,
batch_descriptor=batch_descriptor,
is_padding=input_buffers.is_padding[:num_tokens],
is_padding=is_padding,
):
if cg_mode == CUDAGraphMode.PIECEWISE:
# PIECEWISE graph (compiled PW or breakable, chosen inside
@@ -837,7 +840,7 @@ def prepare_inputs_to_capture(
num_reqs, num_tokens, input_buffers, max_query_len=max_query_len
)
if pcp_manager is not None:
input_batch = pcp_manager.prepare_inputs_to_capture(input_batch)
input_batch = pcp_manager.prepare_inputs_for_capture_or_profile(input_batch)
block_table_provider = pcp_manager or block_tables
input_block_tables = block_table_provider.get_dummy_block_tables(num_reqs)
+12 -9
View File
@@ -305,7 +305,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Draft tokens propagation - for spec-dec + struct outputs.
self.draft_tokens_handler = DraftTokensHandler(self.device)
self.pcp_manager: pcp.PCPManager | None = None
# Built before memory profiling; block tables are attached at KV init.
self.pcp_manager = pcp.maybe_build_pcp_manager(
self.vllm_config,
self.device,
self.supports_mm_inputs,
cls=self.pcp_manager_cls,
)
# Pooling models.
self.is_pooling_model = self.model_config.runner_type == "pooling"
@@ -700,13 +706,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
cp_rank=self.dcp_rank,
cp_interleave=self.cp_interleave,
)
self.pcp_manager = pcp.maybe_build_pcp_manager(
self.vllm_config,
self.device,
self.supports_mm_inputs,
self.block_tables,
cls=self.pcp_manager_cls,
)
if self.pcp_manager is not None:
self.pcp_manager.set_block_tables(self.block_tables)
self.ubatch_runner = maybe_build_ubatch_runner(
self.vllm_config,
self.device,
@@ -1848,7 +1849,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# All-zero input_ids route every token to the same experts.
input_batch.input_ids.random_(0, self.vocab_size)
if self.pcp_manager is not None:
input_batch = self.pcp_manager.prepare_inputs_to_capture(input_batch)
input_batch = self.pcp_manager.prepare_inputs_for_capture_or_profile(
input_batch
)
if skip_attn_for_dummy_run:
assert batch_desc.cg_mode != CUDAGraphMode.FULL, (
"Attention metadata must be prepared for dummy runs when using "
+49 -8
View File
@@ -56,6 +56,7 @@ class PCPManager:
dcp_world_size: int = 1,
dcp_rank: int = 0,
cp_interleave: int = 1,
moe_gathers_tokens: bool = False,
) -> None:
self.pcp_world_size = pcp_world_size
self.pcp_rank = pcp_rank
@@ -69,11 +70,16 @@ class PCPManager:
self._local_batch: InputBatch | None = None
self._local_gather_idx: torch.Tensor | None = None
self.draft_prefill_batch: InputBatch | None = None
self._block_tables = block_tables
self._hidden_restore_idx: torch.Tensor | None = None
self._padded_gather_idx: torch.Tensor | None = None
self._gathered_kv_write_mask: torch.Tensor | None = None
self._pad_slot_id = torch.tensor(PAD_SLOT_ID, dtype=torch.int64, device=device)
# Padding mask over the PCP-gathered rows the MoE routes (rank-major).
self._moe_is_padding = (
torch.ones(max_num_tokens * pcp_world_size, dtype=torch.bool, device=device)
if moe_gathers_tokens and max_num_tokens is not None
else None
)
max_num_local_reqs = 2 * max_num_reqs if max_num_reqs is not None else None
self._input_buffers = (
@@ -81,6 +87,18 @@ class PCPManager:
if max_num_local_reqs is not None and max_num_tokens is not None
else None
)
self._max_num_local_reqs = max_num_local_reqs
self._max_num_tokens = max_num_tokens
self.set_block_tables(block_tables)
def set_block_tables(self, block_tables: BlockTables | None) -> None:
"""Attach the KV block tables, which only exist after KV cache init."""
self._block_tables = block_tables
if block_tables is not None:
self.cp_interleave = block_tables.cp_interleave
max_num_local_reqs = self._max_num_local_reqs
max_num_tokens = self._max_num_tokens
device, pcp_world_size = self.device, self.pcp_world_size
self._local_block_tables: tuple[torch.Tensor, ...] | None
self._local_block_table_ptrs: torch.Tensor | None
if block_tables is not None and max_num_local_reqs is not None:
@@ -593,6 +611,12 @@ class PCPManager:
input_buffers.positions[:num_local_tokens_padded].masked_fill_(
is_padding, 0
)
if self._moe_is_padding is not None:
rows = np.arange(num_local_tokens_padded)
gathered = rows >= np.array(per_rank_num_tokens)[:, None]
is_padding = async_tensor_h2d(
gathered.ravel(), out=self._moe_is_padding[: gathered.size]
)
total_num_logits = num_local_reqs if num_local_tokens > 0 else 0
if total_num_logits > 0:
@@ -683,7 +707,9 @@ class PCPManager:
)
return self._local_batch
def prepare_inputs_to_capture(self, input_batch: InputBatch) -> InputBatch:
def prepare_inputs_for_capture_or_profile(
self, input_batch: InputBatch
) -> InputBatch:
"""Stage a capture or dummy batch in persistent PCP input buffers."""
input_buffers = self.input_buffers
num_reqs = input_batch.num_reqs_after_padding
@@ -692,8 +718,8 @@ class PCPManager:
input_batch,
input_ids=input_buffers.input_ids[:num_tokens].copy_(input_batch.input_ids),
positions=input_buffers.positions[:num_tokens].copy_(input_batch.positions),
is_padding=input_buffers.is_padding[:num_tokens].copy_(
input_batch.is_padding
is_padding=self.moe_is_padding(
input_buffers.is_padding[:num_tokens].copy_(input_batch.is_padding)
),
query_start_loc=input_buffers.query_start_loc[: num_reqs + 1].copy_(
input_batch.query_start_loc
@@ -702,6 +728,15 @@ class PCPManager:
)
return input_batch
def moe_is_padding(self, is_padding: torch.Tensor) -> torch.Tensor:
"""Expand a capture/dummy padding mask, identical on every PCP rank, to
the rows the MoE routes."""
if self._moe_is_padding is None:
return is_padding
out = self._moe_is_padding[: self.pcp_world_size * is_padding.shape[0]]
out.view(self.pcp_world_size, -1).copy_(is_padding)
return out
def get_dummy_block_tables(self, num_reqs: int) -> tuple[torch.Tensor, ...]:
assert self._local_block_tables is not None
return tuple(
@@ -777,9 +812,13 @@ class PCPManager:
return gathered[self._hidden_restore_idx]
def get_draft_input_buffers(
self, input_buffers: InputBuffers
) -> InputBatch | InputBuffers:
return self.draft_prefill_batch or input_buffers
self, input_buffers: InputBuffers, num_tokens: int
) -> tuple[InputBatch | InputBuffers, torch.Tensor]:
"""Return draft inputs and their MoE routing padding mask."""
batch = self.draft_prefill_batch
if batch is not None:
return batch, batch.is_padding
return input_buffers, self.moe_is_padding(input_buffers.is_padding[:num_tokens])
def prepare_draft_prefill(
self, input_batch: InputBatch, input_ids: torch.Tensor
@@ -831,7 +870,7 @@ def maybe_build_pcp_manager(
vllm_config: VllmConfig,
device: torch.device,
supports_mm_inputs: bool,
block_tables: BlockTables,
block_tables: BlockTables | None = None,
cls: type[PCPManager] = PCPManager,
) -> PCPManager | None:
parallel_config = vllm_config.parallel_config
@@ -856,4 +895,6 @@ def maybe_build_pcp_manager(
dcp_world_size=dcp_size,
dcp_rank=dcp_rank,
cp_interleave=parallel_config.cp_kv_cache_interleave_size,
# Without EP all2all kernels, MoERunner all-gathers PCP tokens to route.
moe_gathers_tokens=not parallel_config.enable_expert_parallel,
)
@@ -376,8 +376,9 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
input_buffers: InputBatch | InputBuffers = self.input_buffers
is_padding = None
if self.pcp_manager is not None:
input_buffers = self.pcp_manager.get_draft_input_buffers(self.input_buffers)
is_padding = input_buffers.is_padding[:num_tokens]
input_buffers, is_padding = self.pcp_manager.get_draft_input_buffers(
self.input_buffers, num_tokens
)
batch_descriptor = BatchDescriptor(num_tokens=num_tokens)
with set_forward_context(
attn_metadata,