[MRV2][BugFix] Fix KV connector handling in spec decode case (#43719)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
@@ -168,7 +168,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
slot_mappings_by_layer=None,
|
||||
hidden_states=None,
|
||||
aux_hidden_states=None,
|
||||
kv_connector_output=None,
|
||||
finished_req_ids=set(),
|
||||
num_tokens_across_dp=None,
|
||||
)
|
||||
runner.postprocess = lambda *args, **kwargs: events.append("postprocess")
|
||||
|
||||
@@ -34,7 +34,7 @@ class KVConnector:
|
||||
pass
|
||||
|
||||
def post_forward(
|
||||
self, scheduler_output: "SchedulerOutput", wait_for_save: bool = True
|
||||
self, finished_req_ids: set[str], wait_for_save: bool = True
|
||||
) -> KVConnectorOutput | None:
|
||||
return None
|
||||
|
||||
@@ -76,10 +76,7 @@ class ActiveKVConnector(KVConnector):
|
||||
self.kv_connector.start_load_kv(get_forward_context())
|
||||
|
||||
def post_forward(
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
wait_for_save: bool = True,
|
||||
clear_metadata: bool = True,
|
||||
self, finished_req_ids: set[str], wait_for_save: bool = True
|
||||
) -> KVConnectorOutput | None:
|
||||
if self._disabled:
|
||||
return None
|
||||
@@ -88,7 +85,7 @@ class ActiveKVConnector(KVConnector):
|
||||
if wait_for_save:
|
||||
self.kv_connector.wait_for_save()
|
||||
output.finished_sending, output.finished_recving = (
|
||||
self.kv_connector.get_finished(scheduler_output.finished_req_ids)
|
||||
self.kv_connector.get_finished(finished_req_ids)
|
||||
)
|
||||
output.invalid_block_ids = self.kv_connector.get_block_ids_with_load_errors()
|
||||
output.kv_connector_stats = self.kv_connector.get_kv_connector_stats()
|
||||
@@ -96,9 +93,7 @@ class ActiveKVConnector(KVConnector):
|
||||
output.kv_connector_worker_meta = (
|
||||
self.kv_connector.build_connector_worker_meta()
|
||||
)
|
||||
|
||||
if clear_metadata:
|
||||
self.kv_connector.clear_connector_metadata()
|
||||
self.kv_connector.clear_connector_metadata()
|
||||
return output
|
||||
|
||||
def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
|
||||
@@ -106,7 +101,8 @@ class ActiveKVConnector(KVConnector):
|
||||
return EMPTY_MODEL_RUNNER_OUTPUT
|
||||
|
||||
self.pre_forward(scheduler_output)
|
||||
kv_connector_output = self.post_forward(scheduler_output, wait_for_save=False)
|
||||
finished_req_ids = scheduler_output.finished_req_ids
|
||||
kv_connector_output = self.post_forward(finished_req_ids, wait_for_save=False)
|
||||
if kv_connector_output is None or kv_connector_output.is_empty():
|
||||
return EMPTY_MODEL_RUNNER_OUTPUT
|
||||
output = copy.copy(EMPTY_MODEL_RUNNER_OUTPUT)
|
||||
|
||||
@@ -50,7 +50,7 @@ from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput
|
||||
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
|
||||
from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput
|
||||
from vllm.v1.worker.gpu.attn_utils import (
|
||||
@@ -1207,19 +1207,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
aux_hidden_states = None
|
||||
output_intermediate_tensors = model_output
|
||||
|
||||
kv_connector_output = self.kv_connector.post_forward(scheduler_output)
|
||||
finished_req_ids = scheduler_output.finished_req_ids
|
||||
self.execute_model_state = ExecuteModelState(
|
||||
input_batch=input_batch,
|
||||
attn_metadata=attn_metadata,
|
||||
slot_mappings_by_layer=slot_mappings_by_layer,
|
||||
hidden_states=hidden_states,
|
||||
aux_hidden_states=aux_hidden_states,
|
||||
kv_connector_output=kv_connector_output,
|
||||
finished_req_ids=finished_req_ids,
|
||||
)
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
# Non-last PP rank: return IntermediateTensors for sending.
|
||||
assert output_intermediate_tensors is not None
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
output_intermediate_tensors.kv_connector_output = kv_connector_output
|
||||
return output_intermediate_tensors
|
||||
return None
|
||||
@@ -1238,7 +1239,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
slot_mappings_by_layer = self.execute_model_state.slot_mappings_by_layer
|
||||
hidden_states = self.execute_model_state.hidden_states
|
||||
aux_hidden_states = self.execute_model_state.aux_hidden_states
|
||||
kv_connector_output = self.execute_model_state.kv_connector_output
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
self.execute_model_state = None
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
@@ -1280,8 +1281,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(input_batch.req_ids)},
|
||||
sampled_token_ids=None, # type: ignore
|
||||
prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type]
|
||||
kv_connector_output=kv_connector_output,
|
||||
)
|
||||
# Start async output copy here so that it can overlap with speculator proposal.
|
||||
async_output = AsyncOutput(
|
||||
model_runner_output=model_runner_output,
|
||||
sampler_output=sampler_output,
|
||||
@@ -1344,6 +1345,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
|
||||
self.draft_tokens_handler.set_draft_tokens(input_batch, draft_tokens)
|
||||
|
||||
# Post-step KV connector related operations.
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
model_runner_output.kv_connector_output = kv_connector_output
|
||||
|
||||
if self.use_async_scheduling:
|
||||
return async_output
|
||||
return async_output.get_output()
|
||||
@@ -1360,7 +1365,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
input_batch = self.execute_model_state.input_batch
|
||||
hidden_states = self.execute_model_state.hidden_states
|
||||
kv_connector_output = self.execute_model_state.kv_connector_output
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
self.execute_model_state = None
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
@@ -1372,6 +1377,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
hidden_states, input_batch, self.req_states
|
||||
)
|
||||
|
||||
# Post-step KV connector related operations.
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
|
||||
# Build the model runner output.
|
||||
model_runner_output = ModelRunnerOutput(
|
||||
req_ids=input_batch.req_ids,
|
||||
@@ -1455,4 +1463,4 @@ class ExecuteModelState(NamedTuple):
|
||||
slot_mappings_by_layer: dict[str, torch.Tensor] | None
|
||||
hidden_states: torch.Tensor | None
|
||||
aux_hidden_states: list[torch.Tensor] | None
|
||||
kv_connector_output: KVConnectorOutput | None
|
||||
finished_req_ids: set[str]
|
||||
|
||||
Reference in New Issue
Block a user