[Spec Decode] Defer clearing KV connector metadata for EAGLE3 speculative decode + prefill / decode disagg setup (#34529)

Signed-off-by: qizixi <qizixi@meta.com>
Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
This commit is contained in:
qizixi
2026-02-22 08:08:32 -08:00
committed by GitHub
co-authored by Lu Fang
parent dd8c3a7fb2
commit b9c2a565cc
3 changed files with 38 additions and 5 deletions
+12 -1
View File
@@ -3524,6 +3524,9 @@ class GPUModelRunner(
# Run the model.
# Use persistent buffers for CUDA graphs.
# When spec decode is enabled, delay clearing connector metadata
# until after draft model runs in sample_tokens.
clear_kv_metadata = self.speculative_config is None
with (
set_forward_context(
attn_metadata,
@@ -3537,7 +3540,9 @@ class GPUModelRunner(
skip_compiled=has_encoder_input,
),
record_function_or_nullcontext("gpu_model_runner: forward"),
self.maybe_get_kv_connector_output(scheduler_output) as kv_connector_output,
self.maybe_get_kv_connector_output(
scheduler_output, clear_metadata=clear_kv_metadata
) as kv_connector_output,
):
model_output = self._model_forward(
input_ids=input_ids,
@@ -3765,6 +3770,12 @@ class GPUModelRunner(
# tokens on the CPU, so they are run after bookkeeping.
propose_draft_token_ids(valid_sampled_token_ids)
# Clear KV connector metadata after draft model runs (if spec decode).
# This was deferred from target model forward to allow draft model
# to also save its KV cache.
if self.speculative_config is not None:
self.clear_kv_connector_metadata()
with record_function_or_nullcontext("gpu_model_runner: eplb"):
self.eplb_step()