[Perf][Qwen3.8] Reduce PLE metadata construction overhead (#58114)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
f6877f563c
commit
20b52e9f5b
@@ -672,14 +672,9 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
num_decode_tokens: int,
|
||||
num_prefill_tokens: int,
|
||||
) -> torch.Tensor:
|
||||
# ``non_spec_query_start_loc`` covers the non-spec (decode + prefill)
|
||||
# requests and equals ``query_start_loc`` when spec-decode is inactive.
|
||||
non_spec_query_start_loc = metadata.non_spec_query_start_loc
|
||||
if non_spec_query_start_loc is None:
|
||||
query_start_loc_p = metadata.query_start_loc_p
|
||||
if query_start_loc_p is None:
|
||||
raise ValueError("query_start_loc is required for prefill short-conv")
|
||||
query_start_loc_p = (
|
||||
non_spec_query_start_loc[-num_prefills - 1 :] - num_decode_tokens
|
||||
)
|
||||
# The metadata builder guarantees that the prefill query offsets start
|
||||
# at 0 and end at num_prefill_tokens. Avoid reading those values here,
|
||||
# since doing so would force a device-to-host synchronization.
|
||||
|
||||
@@ -292,10 +292,9 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
token_indices=token_indices_d,
|
||||
)
|
||||
|
||||
query_start_loc = metadata.non_spec_query_start_loc
|
||||
query_start_loc = metadata.query_start_loc_p
|
||||
if query_start_loc is None:
|
||||
raise ValueError("query_start_loc is required for prefill short-conv")
|
||||
query_start_loc = query_start_loc[-num_prefills - 1 :] - num_decode_tokens
|
||||
has_initial_states = metadata.has_initial_states_p
|
||||
if has_initial_states is None:
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user