[Feat] Support PP with PCP in GPU Model Runner V2 (#59139)
Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
parent
2e52a558f0
commit
98b45c1a94
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
import contextlib
|
import contextlib
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import Mock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
@@ -18,7 +19,43 @@ from vllm.v1.kv_cache_interface import (
|
|||||||
UniformTypeKVCacheSpecs,
|
UniformTypeKVCacheSpecs,
|
||||||
)
|
)
|
||||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
from vllm.v1.worker.gpu.model_runner import ExecuteModelState, GPUModelRunner
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_last_pp_rank_uses_global_batch_for_sample_feedback():
|
||||||
|
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||||
|
runner.is_last_pp_rank = False
|
||||||
|
local_batch = object()
|
||||||
|
global_batch = SimpleNamespace(idx_mapping=object())
|
||||||
|
runner.pcp_manager = SimpleNamespace(
|
||||||
|
global_batch=global_batch,
|
||||||
|
restore_for_sampling=Mock(),
|
||||||
|
)
|
||||||
|
runner.pp_handler = SimpleNamespace(receive=Mock(return_value=False))
|
||||||
|
runner.postprocess_num_computed_tokens = Mock()
|
||||||
|
runner.model_state = SimpleNamespace(postprocess_state=Mock())
|
||||||
|
runner.kv_connector = SimpleNamespace(post_forward=Mock(return_value=None))
|
||||||
|
runner.eplb = SimpleNamespace(step=Mock())
|
||||||
|
runner.execute_model_state = ExecuteModelState(
|
||||||
|
input_batch=local_batch,
|
||||||
|
attn_metadata=None,
|
||||||
|
slot_mappings_by_layer=None,
|
||||||
|
hidden_states=None,
|
||||||
|
aux_hidden_states=None,
|
||||||
|
dp_sync=None,
|
||||||
|
finished_req_ids=set(),
|
||||||
|
ec_connector_output=None,
|
||||||
|
cudagraph_stats=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
runner.sample_tokens(None)
|
||||||
|
|
||||||
|
runner.pp_handler.receive.assert_called_once_with(global_batch)
|
||||||
|
runner.postprocess_num_computed_tokens.assert_called_once_with(global_batch)
|
||||||
|
runner.model_state.postprocess_state.assert_called_once_with(
|
||||||
|
global_batch.idx_mapping, 0
|
||||||
|
)
|
||||||
|
runner.pcp_manager.restore_for_sampling.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch):
|
def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch):
|
||||||
|
|||||||
@@ -79,6 +79,7 @@ def _make_runner(**overrides: Any) -> Any:
|
|||||||
runner.use_aux_hidden_state_outputs = False
|
runner.use_aux_hidden_state_outputs = False
|
||||||
runner.speculative_config = None
|
runner.speculative_config = None
|
||||||
runner.speculator = None
|
runner.speculator = None
|
||||||
|
runner.pcp_manager = None
|
||||||
runner.num_speculative_steps = 0
|
runner.num_speculative_steps = 0
|
||||||
runner.encoder_cache = None
|
runner.encoder_cache = None
|
||||||
runner.is_pooling_model = False
|
runner.is_pooling_model = False
|
||||||
|
|||||||
@@ -2064,6 +2064,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
# Non-last PP rank: hidden_states is None because this rank produced
|
# Non-last PP rank: hidden_states is None because this rank produced
|
||||||
# IntermediateTensors instead of final hidden states. Receive the
|
# IntermediateTensors instead of final hidden states. Receive the
|
||||||
# sampled tokens broadcast from the last rank and update local state.
|
# sampled tokens broadcast from the last rank and update local state.
|
||||||
|
if self.pcp_manager is not None:
|
||||||
|
input_batch = self.pcp_manager.global_batch
|
||||||
assert self.pp_handler is not None
|
assert self.pp_handler is not None
|
||||||
all_decode_next = self.pp_handler.receive(input_batch)
|
all_decode_next = self.pp_handler.receive(input_batch)
|
||||||
# Optimistically update num_computed_tokens for entire batch here.
|
# Optimistically update num_computed_tokens for entire batch here.
|
||||||
|
|||||||
@@ -131,8 +131,6 @@ class PCPManager:
|
|||||||
|
|
||||||
if not model_config.use_mla:
|
if not model_config.use_mla:
|
||||||
raise NotImplementedError("MRV2 PCP currently supports MLA models only.")
|
raise NotImplementedError("MRV2 PCP currently supports MLA models only.")
|
||||||
if parallel_config.pipeline_parallel_size > 1:
|
|
||||||
raise NotImplementedError("MRV2 PCP does not support PP yet.")
|
|
||||||
if model_config.is_encoder_decoder:
|
if model_config.is_encoder_decoder:
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"MRV2 PCP does not support encoder-decoder models yet."
|
"MRV2 PCP does not support encoder-decoder models yet."
|
||||||
@@ -406,6 +404,11 @@ class PCPManager:
|
|||||||
assert self._input_buffers is not None
|
assert self._input_buffers is not None
|
||||||
return self._input_buffers
|
return self._input_buffers
|
||||||
|
|
||||||
|
@property
|
||||||
|
def global_batch(self) -> InputBatch:
|
||||||
|
assert self._global_batch is not None
|
||||||
|
return self._global_batch
|
||||||
|
|
||||||
def partition_batch(
|
def partition_batch(
|
||||||
self, input_batch: InputBatch, batch_desc: "BatchExecutionDescriptor"
|
self, input_batch: InputBatch, batch_desc: "BatchExecutionDescriptor"
|
||||||
) -> InputBatch:
|
) -> InputBatch:
|
||||||
|
|||||||
Reference in New Issue
Block a user