[Spec Decode][KV Connector] Fix KV transfer in PD + speculative decoding (#35158)
Signed-off-by: Claude <noreply@anthropic.com> Signed-off-by: Zhanqiu Hu <zh338@cornell.edu> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
parent
807d680337
commit
90f3c01fa4
@@ -3593,9 +3593,9 @@ class GPUModelRunner(
|
||||
|
||||
# Run the model.
|
||||
# Use persistent buffers for CUDA graphs.
|
||||
# When spec decode is enabled, delay clearing connector metadata
|
||||
# until after draft model runs in sample_tokens.
|
||||
clear_kv_metadata = self.speculative_config is None
|
||||
# When spec decode is enabled, defer connector finalization
|
||||
# (wait_for_save + clear metadata) until after draft model runs.
|
||||
defer_kv_connector_finalize = self.speculative_config is not None
|
||||
with (
|
||||
set_forward_context(
|
||||
attn_metadata,
|
||||
@@ -3610,7 +3610,8 @@ class GPUModelRunner(
|
||||
),
|
||||
record_function_or_nullcontext("gpu_model_runner: forward"),
|
||||
self.maybe_get_kv_connector_output(
|
||||
scheduler_output, clear_metadata=clear_kv_metadata
|
||||
scheduler_output,
|
||||
defer_finalize=defer_kv_connector_finalize,
|
||||
) as kv_connector_output,
|
||||
):
|
||||
model_output = self._model_forward(
|
||||
@@ -3843,11 +3844,11 @@ class GPUModelRunner(
|
||||
# tokens on the CPU, so they are run after bookkeeping.
|
||||
propose_draft_token_ids(valid_sampled_token_ids)
|
||||
|
||||
# Clear KV connector metadata after draft model runs (if spec decode).
|
||||
# This was deferred from target model forward to allow draft model
|
||||
# to also save its KV cache.
|
||||
if self.speculative_config is not None:
|
||||
self.clear_kv_connector_metadata()
|
||||
# Finalize KV connector (wait_for_save + clear metadata) after
|
||||
# draft model runs. Deferred from target model forward to allow
|
||||
# draft model to also save its KV cache.
|
||||
if spec_config is not None:
|
||||
self.finalize_kv_connector()
|
||||
|
||||
with record_function_or_nullcontext("gpu_model_runner: eplb"):
|
||||
self.eplb_step()
|
||||
|
||||
@@ -67,16 +67,27 @@ class KVConnectorModelRunnerMixin:
|
||||
@staticmethod
|
||||
def maybe_get_kv_connector_output(
|
||||
scheduler_output: "SchedulerOutput",
|
||||
clear_metadata: bool = True,
|
||||
defer_finalize: bool = False,
|
||||
) -> AbstractContextManager[KVConnectorOutput | None]:
|
||||
return (
|
||||
KVConnectorModelRunnerMixin._get_kv_connector_output(
|
||||
scheduler_output, clear_metadata=clear_metadata
|
||||
scheduler_output, defer_finalize=defer_finalize
|
||||
)
|
||||
if has_kv_transfer_group()
|
||||
else nullcontext()
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def finalize_kv_connector() -> None:
|
||||
"""Finalize the KV connector: wait_for_save and clear metadata.
|
||||
|
||||
Call after draft model forward when defer_finalize=True was used.
|
||||
"""
|
||||
if has_kv_transfer_group():
|
||||
kv_connector = get_kv_transfer_group()
|
||||
kv_connector.wait_for_save()
|
||||
kv_connector.clear_connector_metadata()
|
||||
|
||||
# This context manager must be used within an active forward context.
|
||||
# It encapsulates the entire KV connector lifecycle within execute_model
|
||||
@staticmethod
|
||||
@@ -84,7 +95,7 @@ class KVConnectorModelRunnerMixin:
|
||||
def _get_kv_connector_output(
|
||||
scheduler_output: "SchedulerOutput",
|
||||
wait_for_save: bool = True,
|
||||
clear_metadata: bool = True,
|
||||
defer_finalize: bool = False,
|
||||
) -> Generator[KVConnectorOutput, None, None]:
|
||||
output = KVConnectorOutput()
|
||||
|
||||
@@ -102,7 +113,7 @@ class KVConnectorModelRunnerMixin:
|
||||
try:
|
||||
yield output
|
||||
finally:
|
||||
if wait_for_save:
|
||||
if wait_for_save and not defer_finalize:
|
||||
kv_connector.wait_for_save()
|
||||
|
||||
output.finished_sending, output.finished_recving = (
|
||||
@@ -113,16 +124,9 @@ class KVConnectorModelRunnerMixin:
|
||||
output.kv_connector_stats = kv_connector.get_kv_connector_stats()
|
||||
output.kv_cache_events = kv_connector.get_kv_connector_kv_cache_events()
|
||||
|
||||
if clear_metadata:
|
||||
if not defer_finalize:
|
||||
kv_connector.clear_connector_metadata()
|
||||
|
||||
@staticmethod
|
||||
def clear_kv_connector_metadata() -> None:
|
||||
"""Clear the KV connector metadata. Call after draft model runs."""
|
||||
if has_kv_transfer_group():
|
||||
kv_connector = get_kv_transfer_group()
|
||||
kv_connector.clear_connector_metadata()
|
||||
|
||||
@staticmethod
|
||||
def use_uniform_kv_cache(
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
|
||||
Reference in New Issue
Block a user