[Spec Decode][KV Connector] Fix KV transfer in PD + speculative decoding (#35158)

Signed-off-by: Claude <noreply@anthropic.com>
Signed-off-by: Zhanqiu Hu <zh338@cornell.edu>
Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
zhanqiuhu
2026-03-06 08:50:44 +01:00
committed by GitHub
co-authored by Nicolò Lucchesi
parent 807d680337
commit 90f3c01fa4
5 changed files with 484 additions and 21 deletions
+10 -9
View File
@@ -3593,9 +3593,9 @@ class GPUModelRunner(
# Run the model.
# Use persistent buffers for CUDA graphs.
# When spec decode is enabled, delay clearing connector metadata
# until after draft model runs in sample_tokens.
clear_kv_metadata = self.speculative_config is None
# When spec decode is enabled, defer connector finalization
# (wait_for_save + clear metadata) until after draft model runs.
defer_kv_connector_finalize = self.speculative_config is not None
with (
set_forward_context(
attn_metadata,
@@ -3610,7 +3610,8 @@ class GPUModelRunner(
),
record_function_or_nullcontext("gpu_model_runner: forward"),
self.maybe_get_kv_connector_output(
scheduler_output, clear_metadata=clear_kv_metadata
scheduler_output,
defer_finalize=defer_kv_connector_finalize,
) as kv_connector_output,
):
model_output = self._model_forward(
@@ -3843,11 +3844,11 @@ class GPUModelRunner(
# tokens on the CPU, so they are run after bookkeeping.
propose_draft_token_ids(valid_sampled_token_ids)
# Clear KV connector metadata after draft model runs (if spec decode).
# This was deferred from target model forward to allow draft model
# to also save its KV cache.
if self.speculative_config is not None:
self.clear_kv_connector_metadata()
# Finalize KV connector (wait_for_save + clear metadata) after
# draft model runs. Deferred from target model forward to allow
# draft model to also save its KV cache.
if spec_config is not None:
self.finalize_kv_connector()
with record_function_or_nullcontext("gpu_model_runner: eplb"):
self.eplb_step()
@@ -67,16 +67,27 @@ class KVConnectorModelRunnerMixin:
@staticmethod
def maybe_get_kv_connector_output(
scheduler_output: "SchedulerOutput",
clear_metadata: bool = True,
defer_finalize: bool = False,
) -> AbstractContextManager[KVConnectorOutput | None]:
return (
KVConnectorModelRunnerMixin._get_kv_connector_output(
scheduler_output, clear_metadata=clear_metadata
scheduler_output, defer_finalize=defer_finalize
)
if has_kv_transfer_group()
else nullcontext()
)
@staticmethod
def finalize_kv_connector() -> None:
"""Finalize the KV connector: wait_for_save and clear metadata.
Call after draft model forward when defer_finalize=True was used.
"""
if has_kv_transfer_group():
kv_connector = get_kv_transfer_group()
kv_connector.wait_for_save()
kv_connector.clear_connector_metadata()
# This context manager must be used within an active forward context.
# It encapsulates the entire KV connector lifecycle within execute_model
@staticmethod
@@ -84,7 +95,7 @@ class KVConnectorModelRunnerMixin:
def _get_kv_connector_output(
scheduler_output: "SchedulerOutput",
wait_for_save: bool = True,
clear_metadata: bool = True,
defer_finalize: bool = False,
) -> Generator[KVConnectorOutput, None, None]:
output = KVConnectorOutput()
@@ -102,7 +113,7 @@ class KVConnectorModelRunnerMixin:
try:
yield output
finally:
if wait_for_save:
if wait_for_save and not defer_finalize:
kv_connector.wait_for_save()
output.finished_sending, output.finished_recving = (
@@ -113,16 +124,9 @@ class KVConnectorModelRunnerMixin:
output.kv_connector_stats = kv_connector.get_kv_connector_stats()
output.kv_cache_events = kv_connector.get_kv_connector_kv_cache_events()
if clear_metadata:
if not defer_finalize:
kv_connector.clear_connector_metadata()
@staticmethod
def clear_kv_connector_metadata() -> None:
"""Clear the KV connector metadata. Call after draft model runs."""
if has_kv_transfer_group():
kv_connector = get_kv_transfer_group()
kv_connector.clear_connector_metadata()
@staticmethod
def use_uniform_kv_cache(
attn_groups: list[list[AttentionGroup]],