[Perf][Qwen3.8] Reduce PLE metadata construction overhead (#58114)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Jiangyun Zhu
2026-09-28 20:46:22 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent f6877f563c
commit 20b52e9f5b
5 changed files with 161 additions and 78 deletions
+2 -7
View File
@@ -672,14 +672,9 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
num_decode_tokens: int,
num_prefill_tokens: int,
) -> torch.Tensor:
# ``non_spec_query_start_loc`` covers the non-spec (decode + prefill)
# requests and equals ``query_start_loc`` when spec-decode is inactive.
non_spec_query_start_loc = metadata.non_spec_query_start_loc
if non_spec_query_start_loc is None:
query_start_loc_p = metadata.query_start_loc_p
if query_start_loc_p is None:
raise ValueError("query_start_loc is required for prefill short-conv")
query_start_loc_p = (
non_spec_query_start_loc[-num_prefills - 1 :] - num_decode_tokens
)
# The metadata builder guarantees that the prefill query offsets start
# at 0 and end at num_prefill_tokens. Avoid reading those values here,
# since doing so would force a device-to-host synchronization.
+1 -2
View File
@@ -292,10 +292,9 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
token_indices=token_indices_d,
)
query_start_loc = metadata.non_spec_query_start_loc
query_start_loc = metadata.query_start_loc_p
if query_start_loc is None:
raise ValueError("query_start_loc is required for prefill short-conv")
query_start_loc = query_start_loc[-num_prefills - 1 :] - num_decode_tokens
has_initial_states = metadata.has_initial_states_p
if has_initial_states is None:
raise ValueError(