[Core] Cleanup KVConnector handling with PP + fix MRV2 (#43732)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9090368b65
commit
7e53283b1c
@@ -292,7 +292,7 @@ def test_async_intermediate_tensors_lazy_wait() -> None:
|
||||
)
|
||||
|
||||
# accessing non-tensor attributes should not trigger wait.
|
||||
assert it.kv_connector_output is None
|
||||
assert it._comm_handles is not None
|
||||
assert work.wait_calls == 0
|
||||
assert post_calls["n"] == 0
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
)
|
||||
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||
from vllm.v1.worker.gpu_input_batch import InputBatch
|
||||
@@ -279,7 +280,8 @@ def test_sample_tokens_receives_pp_sampled_ids_only_on_non_last_rank(
|
||||
lambda: SimpleNamespace(world_size=world_size, is_last_rank=is_last_rank),
|
||||
)
|
||||
|
||||
assert GPUModelRunner.sample_tokens(runner, None) is None
|
||||
output = GPUModelRunner.sample_tokens(runner, None)
|
||||
assert output in (EMPTY_MODEL_RUNNER_OUTPUT, None)
|
||||
assert receive_calls == expected_calls
|
||||
|
||||
|
||||
@@ -298,7 +300,8 @@ def test_sample_tokens_skips_pp_group_lookup_without_async_scheduling(
|
||||
pytest.fail,
|
||||
)
|
||||
|
||||
assert GPUModelRunner.sample_tokens(runner, None) is None
|
||||
output = GPUModelRunner.sample_tokens(runner, None)
|
||||
assert output in (EMPTY_MODEL_RUNNER_OUTPUT, None)
|
||||
|
||||
|
||||
def test_select_common_block_size_no_valid_option():
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
|
||||
from vllm.v1.worker.gpu import eplb_utils as eplb
|
||||
from vllm.v1.worker.gpu import model_runner as mrv2
|
||||
|
||||
@@ -76,7 +77,10 @@ def _make_runner(**overrides: Any) -> Any:
|
||||
runner.max_num_reqs = 8
|
||||
runner.max_num_tokens = 16
|
||||
runner.decode_query_len = 1
|
||||
runner.kv_connector = SimpleNamespace(set_disabled=lambda *_: None)
|
||||
runner.kv_connector = SimpleNamespace(
|
||||
set_disabled=lambda *_: None,
|
||||
post_forward=lambda *_, **__: None,
|
||||
)
|
||||
runner.eplb = eplb.EPLBController(runner.parallel_config, runner.device)
|
||||
runner.pooling_runner = None
|
||||
runner.execute_model_state = None
|
||||
@@ -183,5 +187,6 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
),
|
||||
)
|
||||
|
||||
assert mrv2.GPUModelRunner.sample_tokens(runner, None) is None
|
||||
output = mrv2.GPUModelRunner.sample_tokens(runner, None)
|
||||
assert output in (EMPTY_MODEL_RUNNER_OUTPUT, None)
|
||||
assert events == ["postprocess", "eplb"]
|
||||
|
||||
Reference in New Issue
Block a user