[Spec Decode][KV Connector] Fix KV transfer in PD + speculative decoding (#35158)

Signed-off-by: Claude <noreply@anthropic.com>
Signed-off-by: Zhanqiu Hu <zh338@cornell.edu>
Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
zhanqiuhu
2026-03-06 08:50:44 +01:00
committed by GitHub
co-authored by Nicolò Lucchesi
parent 807d680337
commit 90f3c01fa4
5 changed files with 484 additions and 21 deletions
+10 -9
View File
@@ -3593,9 +3593,9 @@ class GPUModelRunner(
# Run the model.
# Use persistent buffers for CUDA graphs.
# When spec decode is enabled, delay clearing connector metadata
# until after draft model runs in sample_tokens.
clear_kv_metadata = self.speculative_config is None
# When spec decode is enabled, defer connector finalization
# (wait_for_save + clear metadata) until after draft model runs.
defer_kv_connector_finalize = self.speculative_config is not None
with (
set_forward_context(
attn_metadata,
@@ -3610,7 +3610,8 @@ class GPUModelRunner(
),
record_function_or_nullcontext("gpu_model_runner: forward"),
self.maybe_get_kv_connector_output(
scheduler_output, clear_metadata=clear_kv_metadata
scheduler_output,
defer_finalize=defer_kv_connector_finalize,
) as kv_connector_output,
):
model_output = self._model_forward(
@@ -3843,11 +3844,11 @@ class GPUModelRunner(
# tokens on the CPU, so they are run after bookkeeping.
propose_draft_token_ids(valid_sampled_token_ids)
# Clear KV connector metadata after draft model runs (if spec decode).
# This was deferred from target model forward to allow draft model
# to also save its KV cache.
if self.speculative_config is not None:
self.clear_kv_connector_metadata()
# Finalize KV connector (wait_for_save + clear metadata) after
# draft model runs. Deferred from target model forward to allow
# draft model to also save its KV cache.
if spec_config is not None:
self.finalize_kv_connector()
with record_function_or_nullcontext("gpu_model_runner: eplb"):
self.eplb_step()