Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -257,7 +257,7 @@ class TestCudagraphDispatcher:
|
||||
def test_get_capture_descs_empty_when_not_initialized(self):
|
||||
"""Test that get_capture_descs returns empty list when keys not initialized."""
|
||||
comp_config = CompilationConfig(
|
||||
cudagraph_mode="FULL",
|
||||
cudagraph_mode=CUDAGraphMode.FULL,
|
||||
mode=CompilationMode.NONE,
|
||||
cudagraph_capture_sizes=[1, 8],
|
||||
)
|
||||
@@ -419,7 +419,7 @@ def _run_and_monitor_call(
|
||||
def test_capture_replay_bypass_logic():
|
||||
comp_config = CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
cudagraph_mode="FULL",
|
||||
cudagraph_mode=CUDAGraphMode.FULL,
|
||||
cudagraph_capture_sizes=[1, 2],
|
||||
)
|
||||
vllm_config = _create_vllm_config(comp_config)
|
||||
@@ -489,7 +489,7 @@ def test_nested_wrappers():
|
||||
"""Tests a scenario with a PIECEWISE wrapper inside a FULL one."""
|
||||
comp_config = CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
cudagraph_mode="FULL",
|
||||
cudagraph_mode=CUDAGraphMode.FULL,
|
||||
cudagraph_capture_sizes=[1],
|
||||
)
|
||||
vllm_config = _create_vllm_config(comp_config)
|
||||
|
||||
@@ -38,7 +38,7 @@ def _reset_graph_pool_id():
|
||||
|
||||
def _create_vllm_config() -> MagicMock:
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode="FULL",
|
||||
cudagraph_mode=CUDAGraphMode.FULL,
|
||||
cudagraph_capture_sizes=[4],
|
||||
)
|
||||
compilation_config.max_cudagraph_capture_size = 4
|
||||
@@ -132,6 +132,7 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings():
|
||||
pcp_world_size=pcp_world_size,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=False,
|
||||
max_num_reqs=num_reqs,
|
||||
max_num_tokens=num_tokens,
|
||||
block_tables=pcp_block_tables,
|
||||
@@ -264,7 +265,7 @@ def _create_decode_vllm_config(
|
||||
use_recoverssm: bool = False,
|
||||
) -> MagicMock:
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode="FULL_AND_PIECEWISE",
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
cudagraph_capture_sizes=capture_sizes,
|
||||
)
|
||||
compilation_config.max_cudagraph_capture_size = capture_sizes[-1]
|
||||
@@ -473,6 +474,7 @@ def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch):
|
||||
assert any(len(descs) > 1 for descs in by_num_tokens.values())
|
||||
|
||||
for desc in full_descs:
|
||||
assert desc.num_reqs is not None
|
||||
assert desc in manager._candidates[(desc.num_tokens, 0)], desc
|
||||
assert (
|
||||
manager.dispatch(
|
||||
|
||||
@@ -14,7 +14,7 @@ Test organization:
|
||||
|
||||
from collections.abc import Hashable
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -100,6 +100,50 @@ class _MockModel(SupportsEncoderCudaGraph):
|
||||
def get_encoder_cudagraph_budget_range(self, vllm_config):
|
||||
return (self._min_budget, self._max_budget)
|
||||
|
||||
def get_max_frames_per_video(self) -> int:
|
||||
return 0
|
||||
|
||||
def get_encoder_cudagraph_item_specs(
|
||||
self, mm_kwargs: dict[str, Any]
|
||||
) -> list[EncoderItemSpec]:
|
||||
return []
|
||||
|
||||
def prepare_encoder_cudagraph_capture_inputs(
|
||||
self,
|
||||
token_budget: int,
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
path: str = "default",
|
||||
axis_keys: tuple[Hashable, ...] | None = None,
|
||||
) -> EncoderCudaGraphCaptureInputs:
|
||||
return EncoderCudaGraphCaptureInputs(values={})
|
||||
|
||||
def prepare_encoder_cudagraph_replay_buffers(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
path: str = "default",
|
||||
) -> EncoderCudaGraphReplayBuffers:
|
||||
return EncoderCudaGraphReplayBuffers(values={})
|
||||
|
||||
def encoder_cudagraph_forward(
|
||||
self, inputs: dict[str, torch.Tensor], path: str = "default"
|
||||
) -> torch.Tensor:
|
||||
return torch.zeros(0)
|
||||
|
||||
def encoder_eager_forward(
|
||||
self, mm_kwargs: dict[str, Any], path: str = "default"
|
||||
) -> torch.Tensor:
|
||||
return torch.zeros(0)
|
||||
|
||||
def select_encoder_cudagraph_items(
|
||||
self, mm_kwargs: dict[str, Any], indices: list[int]
|
||||
) -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
|
||||
def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager:
|
||||
"""Create a minimal EncoderCudaGraphManager with only token_budgets set.
|
||||
@@ -594,6 +638,7 @@ def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only
|
||||
ECConnectorBase,
|
||||
ECConnectorMetadata,
|
||||
)
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.worker.gpu.ec_connector import ActiveECConnector
|
||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
|
||||
@@ -629,8 +674,11 @@ def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only
|
||||
"vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=connector
|
||||
):
|
||||
ec = ActiveECConnector(SimpleNamespace(), cache)
|
||||
scheduled = SimpleNamespace(
|
||||
ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset()
|
||||
scheduled = cast(
|
||||
SchedulerOutput,
|
||||
SimpleNamespace(
|
||||
ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset()
|
||||
),
|
||||
)
|
||||
saved_outputs: list[tuple[torch.Tensor, torch.Tensor]] = []
|
||||
# Mixed sizes exercise packing order; 64 is the boundary, 81 falls back.
|
||||
@@ -643,6 +691,7 @@ def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only
|
||||
outputs = manager.execute(inputs)
|
||||
assert outputs is not None
|
||||
cache[str(len(cache))] = outputs[0]
|
||||
assert ec_output is not None
|
||||
assert ec_output.finished_sending is None
|
||||
assert outputs is not None
|
||||
for actual, eager in zip(outputs, expected):
|
||||
@@ -656,6 +705,7 @@ def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only
|
||||
connector.get_finished.return_value = (set(cache), None)
|
||||
with ec.maybe_get_output(scheduled) as ec_output:
|
||||
pass
|
||||
assert ec_output is not None
|
||||
assert ec_output.finished_sending == set(cache)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user