[Bugfix][PCP] Align MoE routing padding with gathered tokens (#60946)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: wangyicong <wangyicong@bytedance.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
Claude Opus 5.5
OpenAI Codex
parent
8530aed2b7
commit
bfdbfa1632
@@ -56,6 +56,7 @@ class PCPManager:
|
||||
dcp_world_size: int = 1,
|
||||
dcp_rank: int = 0,
|
||||
cp_interleave: int = 1,
|
||||
moe_gathers_tokens: bool = False,
|
||||
) -> None:
|
||||
self.pcp_world_size = pcp_world_size
|
||||
self.pcp_rank = pcp_rank
|
||||
@@ -69,11 +70,16 @@ class PCPManager:
|
||||
self._local_batch: InputBatch | None = None
|
||||
self._local_gather_idx: torch.Tensor | None = None
|
||||
self.draft_prefill_batch: InputBatch | None = None
|
||||
self._block_tables = block_tables
|
||||
self._hidden_restore_idx: torch.Tensor | None = None
|
||||
self._padded_gather_idx: torch.Tensor | None = None
|
||||
self._gathered_kv_write_mask: torch.Tensor | None = None
|
||||
self._pad_slot_id = torch.tensor(PAD_SLOT_ID, dtype=torch.int64, device=device)
|
||||
# Padding mask over the PCP-gathered rows the MoE routes (rank-major).
|
||||
self._moe_is_padding = (
|
||||
torch.ones(max_num_tokens * pcp_world_size, dtype=torch.bool, device=device)
|
||||
if moe_gathers_tokens and max_num_tokens is not None
|
||||
else None
|
||||
)
|
||||
|
||||
max_num_local_reqs = 2 * max_num_reqs if max_num_reqs is not None else None
|
||||
self._input_buffers = (
|
||||
@@ -81,6 +87,18 @@ class PCPManager:
|
||||
if max_num_local_reqs is not None and max_num_tokens is not None
|
||||
else None
|
||||
)
|
||||
self._max_num_local_reqs = max_num_local_reqs
|
||||
self._max_num_tokens = max_num_tokens
|
||||
self.set_block_tables(block_tables)
|
||||
|
||||
def set_block_tables(self, block_tables: BlockTables | None) -> None:
|
||||
"""Attach the KV block tables, which only exist after KV cache init."""
|
||||
self._block_tables = block_tables
|
||||
if block_tables is not None:
|
||||
self.cp_interleave = block_tables.cp_interleave
|
||||
max_num_local_reqs = self._max_num_local_reqs
|
||||
max_num_tokens = self._max_num_tokens
|
||||
device, pcp_world_size = self.device, self.pcp_world_size
|
||||
self._local_block_tables: tuple[torch.Tensor, ...] | None
|
||||
self._local_block_table_ptrs: torch.Tensor | None
|
||||
if block_tables is not None and max_num_local_reqs is not None:
|
||||
@@ -593,6 +611,12 @@ class PCPManager:
|
||||
input_buffers.positions[:num_local_tokens_padded].masked_fill_(
|
||||
is_padding, 0
|
||||
)
|
||||
if self._moe_is_padding is not None:
|
||||
rows = np.arange(num_local_tokens_padded)
|
||||
gathered = rows >= np.array(per_rank_num_tokens)[:, None]
|
||||
is_padding = async_tensor_h2d(
|
||||
gathered.ravel(), out=self._moe_is_padding[: gathered.size]
|
||||
)
|
||||
|
||||
total_num_logits = num_local_reqs if num_local_tokens > 0 else 0
|
||||
if total_num_logits > 0:
|
||||
@@ -683,7 +707,9 @@ class PCPManager:
|
||||
)
|
||||
return self._local_batch
|
||||
|
||||
def prepare_inputs_to_capture(self, input_batch: InputBatch) -> InputBatch:
|
||||
def prepare_inputs_for_capture_or_profile(
|
||||
self, input_batch: InputBatch
|
||||
) -> InputBatch:
|
||||
"""Stage a capture or dummy batch in persistent PCP input buffers."""
|
||||
input_buffers = self.input_buffers
|
||||
num_reqs = input_batch.num_reqs_after_padding
|
||||
@@ -692,8 +718,8 @@ class PCPManager:
|
||||
input_batch,
|
||||
input_ids=input_buffers.input_ids[:num_tokens].copy_(input_batch.input_ids),
|
||||
positions=input_buffers.positions[:num_tokens].copy_(input_batch.positions),
|
||||
is_padding=input_buffers.is_padding[:num_tokens].copy_(
|
||||
input_batch.is_padding
|
||||
is_padding=self.moe_is_padding(
|
||||
input_buffers.is_padding[:num_tokens].copy_(input_batch.is_padding)
|
||||
),
|
||||
query_start_loc=input_buffers.query_start_loc[: num_reqs + 1].copy_(
|
||||
input_batch.query_start_loc
|
||||
@@ -702,6 +728,15 @@ class PCPManager:
|
||||
)
|
||||
return input_batch
|
||||
|
||||
def moe_is_padding(self, is_padding: torch.Tensor) -> torch.Tensor:
|
||||
"""Expand a capture/dummy padding mask, identical on every PCP rank, to
|
||||
the rows the MoE routes."""
|
||||
if self._moe_is_padding is None:
|
||||
return is_padding
|
||||
out = self._moe_is_padding[: self.pcp_world_size * is_padding.shape[0]]
|
||||
out.view(self.pcp_world_size, -1).copy_(is_padding)
|
||||
return out
|
||||
|
||||
def get_dummy_block_tables(self, num_reqs: int) -> tuple[torch.Tensor, ...]:
|
||||
assert self._local_block_tables is not None
|
||||
return tuple(
|
||||
@@ -777,9 +812,13 @@ class PCPManager:
|
||||
return gathered[self._hidden_restore_idx]
|
||||
|
||||
def get_draft_input_buffers(
|
||||
self, input_buffers: InputBuffers
|
||||
) -> InputBatch | InputBuffers:
|
||||
return self.draft_prefill_batch or input_buffers
|
||||
self, input_buffers: InputBuffers, num_tokens: int
|
||||
) -> tuple[InputBatch | InputBuffers, torch.Tensor]:
|
||||
"""Return draft inputs and their MoE routing padding mask."""
|
||||
batch = self.draft_prefill_batch
|
||||
if batch is not None:
|
||||
return batch, batch.is_padding
|
||||
return input_buffers, self.moe_is_padding(input_buffers.is_padding[:num_tokens])
|
||||
|
||||
def prepare_draft_prefill(
|
||||
self, input_batch: InputBatch, input_ids: torch.Tensor
|
||||
@@ -831,7 +870,7 @@ def maybe_build_pcp_manager(
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
supports_mm_inputs: bool,
|
||||
block_tables: BlockTables,
|
||||
block_tables: BlockTables | None = None,
|
||||
cls: type[PCPManager] = PCPManager,
|
||||
) -> PCPManager | None:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
@@ -856,4 +895,6 @@ def maybe_build_pcp_manager(
|
||||
dcp_world_size=dcp_size,
|
||||
dcp_rank=dcp_rank,
|
||||
cp_interleave=parallel_config.cp_kv_cache_interleave_size,
|
||||
# Without EP all2all kernels, MoERunner all-gathers PCP tokens to route.
|
||||
moe_gathers_tokens=not parallel_config.enable_expert_parallel,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user