[Spec Decode] Defer clearing KV connector metadata for EAGLE3 speculative decode + prefill / decode disagg setup (#34529)

Signed-off-by: qizixi <qizixi@meta.com>
Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
This commit is contained in:
qizixi
2026-02-22 08:08:32 -08:00
committed by GitHub
co-authored by Lu Fang
parent dd8c3a7fb2
commit b9c2a565cc
3 changed files with 38 additions and 5 deletions
+11 -2
View File
@@ -77,7 +77,10 @@ class ActiveKVConnector(KVConnector):
self.kv_connector.start_load_kv(get_forward_context())
def post_forward(
self, scheduler_output: "SchedulerOutput", wait_for_save: bool = True
self,
scheduler_output: "SchedulerOutput",
wait_for_save: bool = True,
clear_metadata: bool = True,
) -> KVConnectorOutput | None:
if self._disabled:
return None
@@ -91,9 +94,15 @@ class ActiveKVConnector(KVConnector):
output.invalid_block_ids = self.kv_connector.get_block_ids_with_load_errors()
output.kv_connector_stats = self.kv_connector.get_kv_connector_stats()
output.kv_cache_events = self.kv_connector.get_kv_connector_kv_cache_events()
self.kv_connector.clear_connector_metadata()
if clear_metadata:
self.kv_connector.clear_connector_metadata()
return output
def clear_metadata(self) -> None:
"""Clear the connector metadata. Call this after draft model runs."""
if not self._disabled:
self.kv_connector.clear_connector_metadata()
def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
if self._disabled:
return EMPTY_MODEL_RUNNER_OUTPUT