[EPLB] Mask padding in EPLB load recording (#38128)

Signed-off-by: ilmarkov <markovilya197@gmail.com>
Signed-off-by: Markov Ilya <markovilya19@gmail.com>
Co-authored-by: Markov Ilya <markovilya19@gmail.com>
This commit is contained in:
Ilya Markov
2026-06-28 19:43:58 -07:00
committed by GitHub
co-authored by Markov Ilya
parent 311ad689ad
commit a2abce646f
17 changed files with 289 additions and 14 deletions
@@ -278,6 +278,9 @@ class DFlashSpeculator(DraftModelSpeculator):
self.hidden_states[:num_target_tokens],
self.context_positions[:num_target_tokens],
)
# DFlash processes all speculative tokens in one forward pass,
# so the real token count is num_query_tokens.
self._prepare_eplb_forward(num_query_tokens)
self._generate_draft(
num_reqs,
num_query_tokens,
@@ -354,6 +357,10 @@ class DFlashSpeculator(DraftModelSpeculator):
self.kv_cache_config,
)
# DFlash processes all speculative tokens in one forward pass,
# so the real token count is num_query_tokens.
self._prepare_eplb_forward(num_query_tokens)
if batch_desc.cg_mode == CUDAGraphMode.FULL:
assert self.query_cudagraph_manager is not None
self.query_cudagraph_manager.run_fullgraph(batch_desc)