[EPLB] Mask padding in EPLB load recording (#38128)
Signed-off-by: ilmarkov <markovilya197@gmail.com> Signed-off-by: Markov Ilya <markovilya19@gmail.com> Co-authored-by: Markov Ilya <markovilya19@gmail.com>
This commit is contained in:
co-authored by
Markov Ilya
parent
311ad689ad
commit
a2abce646f
@@ -278,6 +278,9 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.hidden_states[:num_target_tokens],
|
||||
self.context_positions[:num_target_tokens],
|
||||
)
|
||||
# DFlash processes all speculative tokens in one forward pass,
|
||||
# so the real token count is num_query_tokens.
|
||||
self._prepare_eplb_forward(num_query_tokens)
|
||||
self._generate_draft(
|
||||
num_reqs,
|
||||
num_query_tokens,
|
||||
@@ -354,6 +357,10 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.kv_cache_config,
|
||||
)
|
||||
|
||||
# DFlash processes all speculative tokens in one forward pass,
|
||||
# so the real token count is num_query_tokens.
|
||||
self._prepare_eplb_forward(num_query_tokens)
|
||||
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
assert self.query_cudagraph_manager is not None
|
||||
self.query_cudagraph_manager.run_fullgraph(batch_desc)
|
||||
|
||||
Reference in New Issue
Block a user