[Spec Decode] Defer clearing KV connector metadata for EAGLE3 speculative decode + prefill / decode disagg setup (#34529)
Signed-off-by: qizixi <qizixi@meta.com> Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
This commit is contained in:
@@ -3524,6 +3524,9 @@ class GPUModelRunner(
|
||||
|
||||
# Run the model.
|
||||
# Use persistent buffers for CUDA graphs.
|
||||
# When spec decode is enabled, delay clearing connector metadata
|
||||
# until after draft model runs in sample_tokens.
|
||||
clear_kv_metadata = self.speculative_config is None
|
||||
with (
|
||||
set_forward_context(
|
||||
attn_metadata,
|
||||
@@ -3537,7 +3540,9 @@ class GPUModelRunner(
|
||||
skip_compiled=has_encoder_input,
|
||||
),
|
||||
record_function_or_nullcontext("gpu_model_runner: forward"),
|
||||
self.maybe_get_kv_connector_output(scheduler_output) as kv_connector_output,
|
||||
self.maybe_get_kv_connector_output(
|
||||
scheduler_output, clear_metadata=clear_kv_metadata
|
||||
) as kv_connector_output,
|
||||
):
|
||||
model_output = self._model_forward(
|
||||
input_ids=input_ids,
|
||||
@@ -3765,6 +3770,12 @@ class GPUModelRunner(
|
||||
# tokens on the CPU, so they are run after bookkeeping.
|
||||
propose_draft_token_ids(valid_sampled_token_ids)
|
||||
|
||||
# Clear KV connector metadata after draft model runs (if spec decode).
|
||||
# This was deferred from target model forward to allow draft model
|
||||
# to also save its KV cache.
|
||||
if self.speculative_config is not None:
|
||||
self.clear_kv_connector_metadata()
|
||||
|
||||
with record_function_or_nullcontext("gpu_model_runner: eplb"):
|
||||
self.eplb_step()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user