[Model Runner V2] Reserve CUDA graph memory (#53306)
Signed-off-by: Anh Tran <anh.tran.3889@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Anh Tran <anh.tran.3889@gmail.com> Co-authored-by: anhtra3889 <anhtra@nvidia.com>
This commit is contained in:
co-authored by
Anh Tran
anhtra3889
parent
8c2bbe00d5
commit
a4d70bef37
@@ -347,6 +347,41 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
|
|||||||
assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes
|
assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||||
|
"""Cudagraph memory profiling uses a minimal KV cache, so the Mamba
|
||||||
|
block-count guard must only fire for the real cache sizing."""
|
||||||
|
kv_cache_config = SimpleNamespace(has_mamba_layers=True, num_blocks=4)
|
||||||
|
|
||||||
|
compilation_config = CompilationConfig(
|
||||||
|
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||||
|
)
|
||||||
|
with pytest.raises(ValueError, match="exceeds available Mamba cache blocks"):
|
||||||
|
compilation_config.resolve_cudagraph_mode_and_sizes(
|
||||||
|
AttentionCGSupport.ALWAYS,
|
||||||
|
"FakeAttentionBackend",
|
||||||
|
uniform_decode_query_len=1,
|
||||||
|
use_v2_model_runner=True,
|
||||||
|
tensor_parallel_size=1,
|
||||||
|
kv_cache_config=kv_cache_config,
|
||||||
|
max_num_reqs=256,
|
||||||
|
)
|
||||||
|
|
||||||
|
compilation_config = CompilationConfig(
|
||||||
|
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||||
|
)
|
||||||
|
cudagraph_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
|
||||||
|
AttentionCGSupport.ALWAYS,
|
||||||
|
"FakeAttentionBackend",
|
||||||
|
uniform_decode_query_len=1,
|
||||||
|
use_v2_model_runner=True,
|
||||||
|
tensor_parallel_size=1,
|
||||||
|
kv_cache_config=kv_cache_config,
|
||||||
|
max_num_reqs=256,
|
||||||
|
is_profiling=True,
|
||||||
|
)
|
||||||
|
assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("model_config", "expected"),
|
("model_config", "expected"),
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -535,8 +535,9 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode):
|
|||||||
"facebook/opt-125m",
|
"facebook/opt-125m",
|
||||||
max_logprobs=5,
|
max_logprobs=5,
|
||||||
enable_prefix_caching=False,
|
enable_prefix_caching=False,
|
||||||
# 2 other llms alive during whole session
|
# 2 other llms alive during whole session; must also cover the
|
||||||
gpu_memory_utilization=0.05,
|
# cudagraph memory reservation from startup profiling.
|
||||||
|
gpu_memory_utilization=0.1,
|
||||||
max_model_len=16,
|
max_model_len=16,
|
||||||
logprobs_mode=logprobs_mode,
|
logprobs_mode=logprobs_mode,
|
||||||
)
|
)
|
||||||
@@ -575,7 +576,7 @@ def test_prompt_logprobs_mode():
|
|||||||
llm = LLM(
|
llm = LLM(
|
||||||
"facebook/opt-125m",
|
"facebook/opt-125m",
|
||||||
enable_prefix_caching=False,
|
enable_prefix_caching=False,
|
||||||
gpu_memory_utilization=0.05,
|
gpu_memory_utilization=0.1,
|
||||||
max_model_len=16,
|
max_model_len=16,
|
||||||
logprobs_mode=mode,
|
logprobs_mode=mode,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,286 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Unit tests for GPUModelRunner (V2) CUDA graph memory profiling.
|
||||||
|
|
||||||
|
These exercise the orchestration of ``profile_cudagraph_memory`` on CPU by
|
||||||
|
building a runner via ``__new__`` and faking the GPU-only helpers, so the
|
||||||
|
control flow (bootstrap -> sample FULL graphs into a throwaway pool ->
|
||||||
|
extrapolate -> teardown) is covered without a GPU.
|
||||||
|
See https://github.com/vllm-project/vllm/issues/49224.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from vllm.compilation.counter import compilation_counter
|
||||||
|
from vllm.config.compilation import CUDAGraphMode
|
||||||
|
from vllm.v1.worker.gpu import cudagraph_utils as cgu
|
||||||
|
from vllm.v1.worker.gpu import model_runner as mrv2
|
||||||
|
|
||||||
|
GLOBAL_POOL = "global-pool"
|
||||||
|
THROWAWAY_POOL = "throwaway-pool"
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeCudaGraphManager:
|
||||||
|
def __init__(
|
||||||
|
self, needs_capture: bool, num_full_descs: int, piecewise_only: bool = False
|
||||||
|
) -> None:
|
||||||
|
self._needs_capture = needs_capture
|
||||||
|
self.pool: Any = GLOBAL_POOL
|
||||||
|
descs = [object() for _ in range(num_full_descs)]
|
||||||
|
if piecewise_only:
|
||||||
|
self._capture_descs = {CUDAGraphMode.PIECEWISE: descs}
|
||||||
|
else:
|
||||||
|
self._capture_descs = {CUDAGraphMode.FULL: descs} if needs_capture else {}
|
||||||
|
# Profiling hooks set by profile_cudagraph_memory.
|
||||||
|
self._max_full_descs_to_capture: int | None = None
|
||||||
|
self._capture_mem_samples: list[int] | None = None
|
||||||
|
self.use_breakable_cg = False
|
||||||
|
|
||||||
|
def needs_capture(self) -> bool:
|
||||||
|
return self._needs_capture
|
||||||
|
|
||||||
|
|
||||||
|
def _make_profiling_runner(
|
||||||
|
cudagraph_mode: CUDAGraphMode,
|
||||||
|
*,
|
||||||
|
needs_capture: bool = True,
|
||||||
|
num_full_descs: int = 3,
|
||||||
|
piecewise_only: bool = False,
|
||||||
|
captured_bytes: int = 7 << 30,
|
||||||
|
mem_samples: list[int] | None = None,
|
||||||
|
) -> Any:
|
||||||
|
runner: Any = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner)
|
||||||
|
runner.compilation_config = SimpleNamespace(cudagraph_mode=cudagraph_mode)
|
||||||
|
runner.cudagraph_manager = _FakeCudaGraphManager(
|
||||||
|
needs_capture, num_full_descs, piecewise_only
|
||||||
|
)
|
||||||
|
runner.vllm_config = SimpleNamespace()
|
||||||
|
|
||||||
|
events: list[str] = []
|
||||||
|
runner.events = events
|
||||||
|
|
||||||
|
def _capture_model() -> int:
|
||||||
|
events.append("capture")
|
||||||
|
# Simulate the manager's per-FULL-graph memory sampling.
|
||||||
|
samples = runner.cudagraph_manager._capture_mem_samples
|
||||||
|
if samples is not None:
|
||||||
|
samples.extend(mem_samples or [])
|
||||||
|
return captured_bytes
|
||||||
|
|
||||||
|
runner.capture_model = _capture_model
|
||||||
|
return runner
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_module(monkeypatch) -> None:
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def _fake_set_current_vllm_config(_cfg):
|
||||||
|
yield
|
||||||
|
|
||||||
|
monkeypatch.setattr(cgu, "set_current_vllm_config", _fake_set_current_vllm_config)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
cgu,
|
||||||
|
"current_platform",
|
||||||
|
SimpleNamespace(graph_pool_handle=lambda: THROWAWAY_POOL),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
cgu, "_init_minimal_kv_cache_for_profiling", lambda r: r.events.append("init")
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
cgu, "_teardown_profiling_state", lambda r: r.events.append("teardown")
|
||||||
|
)
|
||||||
|
# The profiler reads free GPU memory before/after to compute what it
|
||||||
|
# retained; default to a constant (nothing retained).
|
||||||
|
monkeypatch.setattr(cgu.torch.accelerator, "empty_cache", lambda: None)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
cgu.torch.accelerator, "get_memory_info", lambda: (1 << 30, 1 << 30)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_profile_cudagraph_memory_disabled_returns_zero(monkeypatch):
|
||||||
|
_patch_module(monkeypatch)
|
||||||
|
runner = _make_profiling_runner(CUDAGraphMode.NONE)
|
||||||
|
|
||||||
|
result = cgu.profile_cudagraph_memory(runner)
|
||||||
|
|
||||||
|
assert result == 0
|
||||||
|
# No KV-cache bootstrap or teardown when cudagraphs are disabled.
|
||||||
|
assert runner.events == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_profile_cudagraph_memory_no_graphs_tears_down(monkeypatch):
|
||||||
|
_patch_module(monkeypatch)
|
||||||
|
runner = _make_profiling_runner(CUDAGraphMode.FULL, needs_capture=False)
|
||||||
|
|
||||||
|
result = cgu.profile_cudagraph_memory(runner)
|
||||||
|
|
||||||
|
assert result == 0
|
||||||
|
# Bootstrapped then cleaned up, without capturing or touching the pool.
|
||||||
|
assert runner.events == ["init", "teardown"]
|
||||||
|
assert runner.cudagraph_manager.pool == GLOBAL_POOL
|
||||||
|
|
||||||
|
|
||||||
|
def test_profile_cudagraph_memory_samples_and_extrapolates(monkeypatch):
|
||||||
|
_patch_module(monkeypatch)
|
||||||
|
gib = 1 << 30
|
||||||
|
# Measured delta 1000 MiB includes the sampled FULL graphs (100 + 20 MiB).
|
||||||
|
# Extrapolated FULL cost for 3 graphs: 100 + 2 * 20 = 140 MiB.
|
||||||
|
runner = _make_profiling_runner(
|
||||||
|
CUDAGraphMode.FULL,
|
||||||
|
num_full_descs=3,
|
||||||
|
captured_bytes=1000 * gib,
|
||||||
|
mem_samples=[100 * gib, 20 * gib],
|
||||||
|
)
|
||||||
|
|
||||||
|
result = cgu.profile_cudagraph_memory(runner)
|
||||||
|
|
||||||
|
assert result == (1000 - (100 + 20) + (100 + 2 * 20)) * gib
|
||||||
|
# Bootstrap, capture, and teardown run in order.
|
||||||
|
assert runner.events == ["init", "capture", "teardown"]
|
||||||
|
# Capture must use a throwaway pool, not the persistent global pool.
|
||||||
|
assert runner.cudagraph_manager.pool == THROWAWAY_POOL
|
||||||
|
# FULL capture must be limited to the largest few graphs.
|
||||||
|
assert (
|
||||||
|
runner.cudagraph_manager._max_full_descs_to_capture
|
||||||
|
== cgu._FULL_GRAPH_PROFILING_SAMPLES
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_profile_cudagraph_memory_piecewise_only_returns_measured(monkeypatch):
|
||||||
|
_patch_module(monkeypatch)
|
||||||
|
captured_bytes = 5 << 30
|
||||||
|
runner = _make_profiling_runner(
|
||||||
|
CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||||
|
piecewise_only=True,
|
||||||
|
captured_bytes=captured_bytes,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = cgu.profile_cudagraph_memory(runner)
|
||||||
|
|
||||||
|
# No FULL graphs to sample or extrapolate: the measured delta is exact.
|
||||||
|
assert result == captured_bytes
|
||||||
|
|
||||||
|
|
||||||
|
def test_profile_cudagraph_memory_tears_down_on_capture_error(monkeypatch):
|
||||||
|
_patch_module(monkeypatch)
|
||||||
|
runner = _make_profiling_runner(CUDAGraphMode.FULL)
|
||||||
|
|
||||||
|
def _boom() -> int:
|
||||||
|
runner.events.append("capture")
|
||||||
|
raise RuntimeError("capture failed")
|
||||||
|
|
||||||
|
runner.capture_model = _boom
|
||||||
|
|
||||||
|
try:
|
||||||
|
cgu.profile_cudagraph_memory(runner)
|
||||||
|
except RuntimeError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
raise AssertionError("expected capture error to propagate")
|
||||||
|
|
||||||
|
# Teardown still runs even if capture raises.
|
||||||
|
assert runner.events == ["init", "capture", "teardown"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_profile_cudagraph_memory_restores_compilation_counters(monkeypatch):
|
||||||
|
_patch_module(monkeypatch)
|
||||||
|
runner = _make_profiling_runner(CUDAGraphMode.FULL)
|
||||||
|
|
||||||
|
def _capture_model() -> int:
|
||||||
|
compilation_counter.num_cudagraph_captured += 5
|
||||||
|
compilation_counter.num_gpu_runner_capture_triggers += 1
|
||||||
|
return 1 << 30
|
||||||
|
|
||||||
|
runner.capture_model = _capture_model
|
||||||
|
captured_before = compilation_counter.num_cudagraph_captured
|
||||||
|
triggers_before = compilation_counter.num_gpu_runner_capture_triggers
|
||||||
|
|
||||||
|
cgu.profile_cudagraph_memory(runner)
|
||||||
|
|
||||||
|
# Profiling captures are discarded, so they must not inflate the
|
||||||
|
# compilation counters; the real capture_model() runs later.
|
||||||
|
assert compilation_counter.num_cudagraph_captured == captured_before
|
||||||
|
assert compilation_counter.num_gpu_runner_capture_triggers == triggers_before
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_runner_delegates_to_cudagraph_utils(monkeypatch):
|
||||||
|
runner = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner)
|
||||||
|
monkeypatch.setattr(mrv2, "_profile_cudagraph_memory", lambda r: 42)
|
||||||
|
assert runner.profile_cudagraph_memory() == 42
|
||||||
|
|
||||||
|
|
||||||
|
def test_extrapolate_full_graph_memory():
|
||||||
|
mib = 1 << 20
|
||||||
|
# No samples (e.g. no FULL graphs): nothing to add.
|
||||||
|
assert cgu._extrapolate_full_graph_memory([], 0) == 0
|
||||||
|
# A single graph costs exactly its sample.
|
||||||
|
assert cgu._extrapolate_full_graph_memory([100 * mib], 1) == 100 * mib
|
||||||
|
# First capture + per-graph cost for the rest.
|
||||||
|
assert (
|
||||||
|
cgu._extrapolate_full_graph_memory([100 * mib, 20 * mib], 5)
|
||||||
|
== (100 + 4 * 20) * mib
|
||||||
|
)
|
||||||
|
# Per-graph cost is floored to account for driver overhead.
|
||||||
|
assert cgu._extrapolate_full_graph_memory([100 * mib, 0], 3) == (100 + 2 * 1) * mib
|
||||||
|
|
||||||
|
|
||||||
|
def test_profile_cudagraph_memory_clears_captured_graphs(monkeypatch):
|
||||||
|
_patch_module(monkeypatch)
|
||||||
|
runner = _make_profiling_runner(CUDAGraphMode.FULL_AND_PIECEWISE)
|
||||||
|
|
||||||
|
cleared: list[str] = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
cgu.CUDAGraphWrapper,
|
||||||
|
"clear_all_graphs",
|
||||||
|
classmethod(lambda cls: cleared.append("piecewise")),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
cgu.BreakableCUDAGraphWrapper,
|
||||||
|
"clear_all_graphs",
|
||||||
|
classmethod(lambda cls: cleared.append("breakable")),
|
||||||
|
)
|
||||||
|
|
||||||
|
cgu.profile_cudagraph_memory(runner)
|
||||||
|
|
||||||
|
# Profiling captures are discarded so the real capture re-captures them
|
||||||
|
# against the KV cache.
|
||||||
|
assert cleared == ["piecewise", "breakable"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_profile_cudagraph_memory_redirects_wrapper_pools(monkeypatch):
|
||||||
|
"""Piecewise wrappers must capture into the throwaway pool too.
|
||||||
|
|
||||||
|
Profiling graphs captured into the persistent global pool and then
|
||||||
|
discarded drop the pool's use_count to 0, tripping the c10 allocator's
|
||||||
|
create_or_incref_pool assert when the real capture reuses that pool.
|
||||||
|
"""
|
||||||
|
_patch_module(monkeypatch)
|
||||||
|
runner = _make_profiling_runner(CUDAGraphMode.FULL_AND_PIECEWISE)
|
||||||
|
|
||||||
|
class _FakeWrapper:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.graph_pool: Any = GLOBAL_POOL
|
||||||
|
self.pool_during_capture: Any = None
|
||||||
|
|
||||||
|
def clear_graphs(self) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
wrapper = _FakeWrapper()
|
||||||
|
cgu.CUDAGraphWrapper._all_instances.add(wrapper)
|
||||||
|
try:
|
||||||
|
capture_model = runner.capture_model
|
||||||
|
|
||||||
|
def _capture_model() -> int:
|
||||||
|
wrapper.pool_during_capture = wrapper.graph_pool
|
||||||
|
return capture_model()
|
||||||
|
|
||||||
|
runner.capture_model = _capture_model
|
||||||
|
|
||||||
|
cgu.profile_cudagraph_memory(runner)
|
||||||
|
|
||||||
|
assert wrapper.pool_during_capture == THROWAWAY_POOL
|
||||||
|
assert wrapper.graph_pool == GLOBAL_POOL
|
||||||
|
finally:
|
||||||
|
cgu.CUDAGraphWrapper._all_instances.discard(wrapper)
|
||||||
@@ -416,7 +416,10 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert isinstance(attn_metadata_raw, dict)
|
assert isinstance(attn_metadata_raw, dict)
|
||||||
attn_metadata_narrowed = attn_metadata_raw[self.prefix]
|
attn_metadata_narrowed = attn_metadata_raw.get(self.prefix)
|
||||||
|
if attn_metadata_narrowed is None:
|
||||||
|
# Profile/warmup dummy runs skip mamba-family metadata.
|
||||||
|
return
|
||||||
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
|
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
|
||||||
m = attn_metadata_narrowed
|
m = attn_metadata_narrowed
|
||||||
has_initial_state = m.has_initial_state
|
has_initial_state = m.has_initial_state
|
||||||
|
|||||||
@@ -305,7 +305,10 @@ class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
return
|
return
|
||||||
|
|
||||||
assert isinstance(attn_metadata, dict)
|
assert isinstance(attn_metadata, dict)
|
||||||
attn_metadata = attn_metadata[self.prefix] # type: ignore[assignment]
|
attn_metadata = attn_metadata.get(self.prefix) # type: ignore[assignment]
|
||||||
|
if attn_metadata is None:
|
||||||
|
# Profile/warmup dummy runs skip mamba-family metadata.
|
||||||
|
return
|
||||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||||
has_initial_state = attn_metadata.has_initial_state
|
has_initial_state = attn_metadata.has_initial_state
|
||||||
spec_query_start_loc = attn_metadata.spec_query_start_loc
|
spec_query_start_loc = attn_metadata.spec_query_start_loc
|
||||||
|
|||||||
@@ -1202,13 +1202,14 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
forward_context = get_forward_context()
|
forward_context = get_forward_context()
|
||||||
attn_metadata_raw = forward_context.attn_metadata
|
attn_metadata_raw = forward_context.attn_metadata
|
||||||
|
|
||||||
if attn_metadata_raw is None:
|
attn_metadata = None
|
||||||
|
if isinstance(attn_metadata_raw, dict):
|
||||||
|
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||||
|
if attn_metadata is None:
|
||||||
v_dim = core_attn_out.shape[-1] * core_attn_out.shape[-2]
|
v_dim = core_attn_out.shape[-1] * core_attn_out.shape[-2]
|
||||||
self._warmup_prefill_kernels(qkvz, v_dim)
|
self._warmup_prefill_kernels(qkvz, v_dim)
|
||||||
return
|
return
|
||||||
|
|
||||||
assert isinstance(attn_metadata_raw, dict)
|
|
||||||
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
|
|
||||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||||
|
|
||||||
# The AITER fused reshape/conv kernel expects Qwen3-Next's interleaved
|
# The AITER fused reshape/conv kernel expects Qwen3-Next's interleaved
|
||||||
@@ -1259,12 +1260,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
forward_context = get_forward_context()
|
forward_context = get_forward_context()
|
||||||
attn_metadata_raw = forward_context.attn_metadata
|
attn_metadata_raw = forward_context.attn_metadata
|
||||||
|
|
||||||
if attn_metadata_raw is None:
|
attn_metadata = None
|
||||||
|
if isinstance(attn_metadata_raw, dict):
|
||||||
|
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||||
|
if attn_metadata is None:
|
||||||
self._warmup_prefill_kernels(mixed_qkv, 0)
|
self._warmup_prefill_kernels(mixed_qkv, 0)
|
||||||
return
|
return
|
||||||
|
|
||||||
assert isinstance(attn_metadata_raw, dict)
|
|
||||||
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
|
|
||||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||||
|
|
||||||
if (
|
if (
|
||||||
@@ -1773,12 +1775,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
forward_context = get_forward_context()
|
forward_context = get_forward_context()
|
||||||
attn_metadata_raw = forward_context.attn_metadata
|
attn_metadata_raw = forward_context.attn_metadata
|
||||||
qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size
|
qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size
|
||||||
if attn_metadata_raw is None:
|
attn_metadata = None
|
||||||
|
if isinstance(attn_metadata_raw, dict):
|
||||||
|
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||||
|
if attn_metadata is None:
|
||||||
self._warmup_prefill_kernels(mixed_qkvz[:, :qkv_size], 0)
|
self._warmup_prefill_kernels(mixed_qkvz[:, :qkv_size], 0)
|
||||||
return
|
return
|
||||||
|
|
||||||
assert isinstance(attn_metadata_raw, dict)
|
|
||||||
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
|
|
||||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||||
mixed_qkv, output_gate_flat = mixed_qkvz.split(
|
mixed_qkv, output_gate_flat = mixed_qkvz.split(
|
||||||
[qkv_size, self.value_dim // self.tp_size], dim=-1
|
[qkv_size, self.value_dim // self.tp_size], dim=-1
|
||||||
@@ -1856,12 +1859,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
|||||||
) -> None:
|
) -> None:
|
||||||
forward_context = get_forward_context()
|
forward_context = get_forward_context()
|
||||||
attn_metadata_raw = forward_context.attn_metadata
|
attn_metadata_raw = forward_context.attn_metadata
|
||||||
if attn_metadata_raw is None:
|
attn_metadata = None
|
||||||
|
if isinstance(attn_metadata_raw, dict):
|
||||||
|
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||||
|
if attn_metadata is None:
|
||||||
self._warmup_prefill_kernels(mixed_qkv, 0)
|
self._warmup_prefill_kernels(mixed_qkv, 0)
|
||||||
return
|
return
|
||||||
|
|
||||||
assert isinstance(attn_metadata_raw, dict)
|
|
||||||
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
|
|
||||||
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
assert isinstance(attn_metadata, GDNAttentionMetadata)
|
||||||
if (
|
if (
|
||||||
self._can_use_fused_gdn_mtp_decode(attn_metadata)
|
self._can_use_fused_gdn_mtp_decode(attn_metadata)
|
||||||
|
|||||||
@@ -542,7 +542,8 @@ class BailingMoELinearAttention(LinearAttention):
|
|||||||
attn_metadata = forward_context.attn_metadata
|
attn_metadata = forward_context.attn_metadata
|
||||||
if attn_metadata is not None:
|
if attn_metadata is not None:
|
||||||
assert isinstance(attn_metadata, dict)
|
assert isinstance(attn_metadata, dict)
|
||||||
attn_metadata = attn_metadata[self.prefix] # type: ignore
|
attn_metadata = attn_metadata.get(self.prefix) # type: ignore
|
||||||
|
if attn_metadata is not None:
|
||||||
assert isinstance(attn_metadata, LinearAttentionMetadata)
|
assert isinstance(attn_metadata, LinearAttentionMetadata)
|
||||||
num_actual_tokens = (
|
num_actual_tokens = (
|
||||||
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
|
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
|
||||||
|
|||||||
@@ -282,7 +282,8 @@ class MiniMaxText01LinearAttention(LinearAttention):
|
|||||||
attn_metadata: AttentionMetadata | None = None
|
attn_metadata: AttentionMetadata | None = None
|
||||||
if attn_metadata_raw is not None:
|
if attn_metadata_raw is not None:
|
||||||
assert isinstance(attn_metadata_raw, dict)
|
assert isinstance(attn_metadata_raw, dict)
|
||||||
attn_metadata = attn_metadata_raw[self.prefix]
|
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||||
|
if attn_metadata is not None:
|
||||||
assert isinstance(attn_metadata, LinearAttentionMetadata)
|
assert isinstance(attn_metadata, LinearAttentionMetadata)
|
||||||
num_actual_tokens = (
|
num_actual_tokens = (
|
||||||
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
|
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
|
||||||
|
|||||||
@@ -269,7 +269,8 @@ class MambaMixer(MambaBase, PluggableLayer):
|
|||||||
attn_metadata: AttentionMetadata | None = None
|
attn_metadata: AttentionMetadata | None = None
|
||||||
if attn_metadata_raw is not None:
|
if attn_metadata_raw is not None:
|
||||||
assert isinstance(attn_metadata_raw, dict)
|
assert isinstance(attn_metadata_raw, dict)
|
||||||
attn_metadata = attn_metadata_raw[self.prefix]
|
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||||
|
if attn_metadata is not None:
|
||||||
assert isinstance(attn_metadata, Mamba1AttentionMetadata)
|
assert isinstance(attn_metadata, Mamba1AttentionMetadata)
|
||||||
query_start_loc_p = attn_metadata.query_start_loc_p
|
query_start_loc_p = attn_metadata.query_start_loc_p
|
||||||
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
|
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
|
||||||
|
|||||||
@@ -710,7 +710,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
|||||||
attn_metadata: AttentionMetadata | None = None
|
attn_metadata: AttentionMetadata | None = None
|
||||||
if attn_metadata_raw is not None:
|
if attn_metadata_raw is not None:
|
||||||
assert isinstance(attn_metadata_raw, dict)
|
assert isinstance(attn_metadata_raw, dict)
|
||||||
attn_metadata = attn_metadata_raw[self.prefix]
|
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||||
|
if attn_metadata is not None:
|
||||||
assert isinstance(attn_metadata, Mamba2AttentionMetadata)
|
assert isinstance(attn_metadata, Mamba2AttentionMetadata)
|
||||||
# conv_state must be (..., dim, width-1) for the conv kernels.
|
# conv_state must be (..., dim, width-1) for the conv kernels.
|
||||||
# DS layout stores it that way directly; SD layout needs a
|
# DS layout stores it that way directly; SD layout needs a
|
||||||
|
|||||||
@@ -113,8 +113,10 @@ class ShortConv(MambaBase, PluggableLayer):
|
|||||||
attn_metadata: AttentionMetadata | None = None
|
attn_metadata: AttentionMetadata | None = None
|
||||||
if attn_metadata_raw is not None:
|
if attn_metadata_raw is not None:
|
||||||
assert isinstance(attn_metadata_raw, dict)
|
assert isinstance(attn_metadata_raw, dict)
|
||||||
attn_metadata = attn_metadata_raw[self.prefix]
|
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||||
assert isinstance(attn_metadata, ShortConvAttentionMetadata)
|
assert attn_metadata is None or isinstance(
|
||||||
|
attn_metadata, ShortConvAttentionMetadata
|
||||||
|
)
|
||||||
|
|
||||||
BCx, _ = self.in_proj(hidden_states)
|
BCx, _ = self.in_proj(hidden_states)
|
||||||
B, C, x = BCx.chunk(3, dim=-1)
|
B, C, x = BCx.chunk(3, dim=-1)
|
||||||
@@ -225,7 +227,8 @@ class ShortConv(MambaBase, PluggableLayer):
|
|||||||
attn_metadata: AttentionMetadata | None = None
|
attn_metadata: AttentionMetadata | None = None
|
||||||
if attn_metadata_raw is not None:
|
if attn_metadata_raw is not None:
|
||||||
assert isinstance(attn_metadata_raw, dict)
|
assert isinstance(attn_metadata_raw, dict)
|
||||||
attn_metadata = attn_metadata_raw[self.prefix]
|
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||||
|
if attn_metadata is not None:
|
||||||
assert isinstance(attn_metadata, ShortConvAttentionMetadata)
|
assert isinstance(attn_metadata, ShortConvAttentionMetadata)
|
||||||
conv_state = (
|
conv_state = (
|
||||||
self.kv_cache[0]
|
self.kv_cache[0]
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
import gc
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from itertools import product
|
from itertools import product
|
||||||
from typing import Any, NamedTuple, Protocol
|
from typing import TYPE_CHECKING, Any, NamedTuple, Protocol
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
@@ -15,7 +16,8 @@ from vllm.compilation.breakable_cudagraph import (
|
|||||||
is_breakable_cudagraph_enabled,
|
is_breakable_cudagraph_enabled,
|
||||||
)
|
)
|
||||||
from vllm.compilation.counter import compilation_counter
|
from vllm.compilation.counter import compilation_counter
|
||||||
from vllm.config import VllmConfig
|
from vllm.compilation.cuda_graph import CUDAGraphWrapper
|
||||||
|
from vllm.config import VllmConfig, set_current_vllm_config
|
||||||
from vllm.config.compilation import CUDAGraphMode
|
from vllm.config.compilation import CUDAGraphMode
|
||||||
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
|
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
|
||||||
from vllm.distributed.parallel_state import (
|
from vllm.distributed.parallel_state import (
|
||||||
@@ -37,6 +39,9 @@ from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
|||||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||||
from vllm.v1.worker.utils import AttentionGroup
|
from vllm.v1.worker.utils import AttentionGroup
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@@ -132,6 +137,12 @@ class CudaGraphManager:
|
|||||||
|
|
||||||
self._graphs_captured = False
|
self._graphs_captured = False
|
||||||
|
|
||||||
|
# Profiling hooks, set only by profile_cudagraph_memory() below: cap
|
||||||
|
# FULL-mode capture at the N largest descriptors and record each
|
||||||
|
# captured FULL graph's memory delta for extrapolation.
|
||||||
|
self._max_full_descs_to_capture: int | None = None
|
||||||
|
self._capture_mem_samples: list[int] | None = None
|
||||||
|
|
||||||
self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {}
|
self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {}
|
||||||
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
|
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
|
||||||
|
|
||||||
@@ -327,6 +338,14 @@ class CudaGraphManager:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
descs = self._capture_descs[mode]
|
descs = self._capture_descs[mode]
|
||||||
|
if (
|
||||||
|
mode == CUDAGraphMode.FULL
|
||||||
|
and self._max_full_descs_to_capture is not None
|
||||||
|
):
|
||||||
|
# Profiling only: capture a sample of the largest FULL
|
||||||
|
# graphs; the total cost is extrapolated from their
|
||||||
|
# per-graph memory deltas.
|
||||||
|
descs = descs[: self._max_full_descs_to_capture]
|
||||||
if is_global_first_rank():
|
if is_global_first_rank():
|
||||||
descs = tqdm(descs, desc=f"{progress_bar_desc} ({mode.name})")
|
descs = tqdm(descs, desc=f"{progress_bar_desc} ({mode.name})")
|
||||||
for desc in descs:
|
for desc in descs:
|
||||||
@@ -362,6 +381,9 @@ class CudaGraphManager:
|
|||||||
set_graph_pool_id(self.pool)
|
set_graph_pool_id(self.pool)
|
||||||
else:
|
else:
|
||||||
set_graph_pool_id(current_platform.graph_pool_handle())
|
set_graph_pool_id(current_platform.graph_pool_handle())
|
||||||
|
if self._capture_mem_samples is not None:
|
||||||
|
torch.accelerator.synchronize()
|
||||||
|
free_before = torch.accelerator.get_memory_info()[0]
|
||||||
with torch.cuda.graph(graph, self.pool):
|
with torch.cuda.graph(graph, self.pool):
|
||||||
forward_fn(CUDAGraphMode.NONE)
|
forward_fn(CUDAGraphMode.NONE)
|
||||||
# Join offloader's copy stream after forward to avoid
|
# Join offloader's copy stream after forward to avoid
|
||||||
@@ -369,6 +391,10 @@ class CudaGraphManager:
|
|||||||
# forks copy_stream, but wait_prefetch only happens in
|
# forks copy_stream, but wait_prefetch only happens in
|
||||||
# the next forward pass.
|
# the next forward pass.
|
||||||
get_offloader().join_after_forward()
|
get_offloader().join_after_forward()
|
||||||
|
if self._capture_mem_samples is not None:
|
||||||
|
torch.accelerator.synchronize()
|
||||||
|
free_after = torch.accelerator.get_memory_info()[0]
|
||||||
|
self._capture_mem_samples.append(free_before - free_after)
|
||||||
self.graphs[desc] = graph
|
self.graphs[desc] = graph
|
||||||
compilation_counter.num_cudagraph_captured += 1
|
compilation_counter.num_cudagraph_captured += 1
|
||||||
self._graphs_captured = True
|
self._graphs_captured = True
|
||||||
@@ -668,3 +694,162 @@ def prepare_inputs_to_capture(
|
|||||||
for_capture=full_cudagraph,
|
for_capture=full_cudagraph,
|
||||||
)
|
)
|
||||||
return AttentionState(attn_metadata, slot_mappings_by_layer)
|
return AttentionState(attn_metadata, slot_mappings_by_layer)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# CUDA graph memory profiling
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Number of FULL graphs captured during profiling; the total FULL capture
|
||||||
|
# cost is extrapolated from this sample to avoid a second full capture.
|
||||||
|
_FULL_GRAPH_PROFILING_SAMPLES = 2
|
||||||
|
# Floor for the extrapolated per-graph cost (driver overhead per graph).
|
||||||
|
_MIN_PER_GRAPH_BYTES = 1 << 20
|
||||||
|
|
||||||
|
|
||||||
|
@torch.inference_mode()
|
||||||
|
def profile_cudagraph_memory(runner: "GPUModelRunner") -> int:
|
||||||
|
"""Estimate the GPU memory needed for CUDA graph capture.
|
||||||
|
|
||||||
|
Called during memory profiling, *before* the real KV cache is allocated,
|
||||||
|
so that ``Worker.determine_available_memory`` can reserve headroom for
|
||||||
|
graph capture. Bootstraps a minimal KV cache, runs ``capture_model()``
|
||||||
|
once, then releases everything so the real init/capture path starts clean.
|
||||||
|
|
||||||
|
FULL graphs bake in KV cache pointers, so only the largest few are
|
||||||
|
captured (into a throwaway pool) and their total cost is extrapolated.
|
||||||
|
PIECEWISE, encoder and speculator graphs are measured in full. All
|
||||||
|
profiling captures are discarded afterwards: replaying graphs recorded
|
||||||
|
against the throwaway profiling state is unsafe (e.g. inductor graph
|
||||||
|
partition reclaims the storages of earlier cudagraph recordings once the
|
||||||
|
real capture records new ones, leading to use-after-free crashes).
|
||||||
|
"""
|
||||||
|
if runner.compilation_config.cudagraph_mode == CUDAGraphMode.NONE:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
gc.collect()
|
||||||
|
torch.accelerator.empty_cache()
|
||||||
|
|
||||||
|
with set_current_vllm_config(runner.vllm_config):
|
||||||
|
_init_minimal_kv_cache_for_profiling(runner)
|
||||||
|
|
||||||
|
manager = runner.cudagraph_manager
|
||||||
|
assert manager is not None
|
||||||
|
|
||||||
|
# Don't count profiling captures; the real capture_model() runs later.
|
||||||
|
saved_num_cudagraph_captured = compilation_counter.num_cudagraph_captured
|
||||||
|
saved_capture_triggers = compilation_counter.num_gpu_runner_capture_triggers
|
||||||
|
all_wrappers: list[Any] = []
|
||||||
|
original_pools: dict[int, Any] = {}
|
||||||
|
try:
|
||||||
|
if not manager.needs_capture():
|
||||||
|
return 0
|
||||||
|
# Capture all profiling graphs into a throwaway pool so their memory
|
||||||
|
# is reclaimed on teardown rather than retained by the persistent
|
||||||
|
# global pool (which the real capture reuses). This must include the
|
||||||
|
# piecewise wrappers, not just the FULL-graph manager pool: graphs
|
||||||
|
# captured into the global pool and then discarded drop its use_count
|
||||||
|
# to 0, and the real capture on the same pool trips the c10 allocator's
|
||||||
|
# create_or_incref_pool assert ("use_count > 0 INTERNAL ASSERT FAILED").
|
||||||
|
manager.pool = current_platform.graph_pool_handle()
|
||||||
|
if manager.use_breakable_cg:
|
||||||
|
# The breakable runner is otherwise created lazily during capture,
|
||||||
|
# after the pool swap below, and would capture into the global
|
||||||
|
# pool. Create it now so its pool gets swapped too.
|
||||||
|
manager.init_breakable_cg_runner(runner.model)
|
||||||
|
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
|
||||||
|
BreakableCUDAGraphWrapper._all_instances
|
||||||
|
)
|
||||||
|
for wrapper in all_wrappers:
|
||||||
|
original_pools[id(wrapper)] = wrapper.graph_pool
|
||||||
|
wrapper.graph_pool = manager.pool
|
||||||
|
manager._max_full_descs_to_capture = _FULL_GRAPH_PROFILING_SAMPLES
|
||||||
|
mem_samples: list[int] = []
|
||||||
|
manager._capture_mem_samples = mem_samples
|
||||||
|
|
||||||
|
measured = int(runner.capture_model())
|
||||||
|
|
||||||
|
# The measured delta covers PIECEWISE, encoder and speculator graphs
|
||||||
|
# plus the sampled FULL graphs; swap the sampled FULL cost for the
|
||||||
|
# extrapolated total. FULL and PIECEWISE share one pool here just as
|
||||||
|
# they share the global pool at runtime, so the overlap is not
|
||||||
|
# double-counted.
|
||||||
|
num_full_graphs = len(manager._capture_descs.get(CUDAGraphMode.FULL, []))
|
||||||
|
full_estimate = _extrapolate_full_graph_memory(mem_samples, num_full_graphs)
|
||||||
|
return max(measured - sum(mem_samples) + full_estimate, 0)
|
||||||
|
finally:
|
||||||
|
compilation_counter.num_cudagraph_captured = saved_num_cudagraph_captured
|
||||||
|
compilation_counter.num_gpu_runner_capture_triggers = saved_capture_triggers
|
||||||
|
CUDAGraphWrapper.clear_all_graphs()
|
||||||
|
BreakableCUDAGraphWrapper.clear_all_graphs()
|
||||||
|
for wrapper in all_wrappers:
|
||||||
|
if id(wrapper) in original_pools:
|
||||||
|
wrapper.graph_pool = original_pools[id(wrapper)]
|
||||||
|
_teardown_profiling_state(runner)
|
||||||
|
|
||||||
|
|
||||||
|
def _extrapolate_full_graph_memory(mem_samples: list[int], total_graphs: int) -> int:
|
||||||
|
"""Extrapolate the total FULL capture cost from samples of the largest
|
||||||
|
graphs. The first capture allocates the pool baseline; later graphs mostly
|
||||||
|
reuse it, so the second sample is taken as the per-graph cost."""
|
||||||
|
if not mem_samples:
|
||||||
|
return 0
|
||||||
|
first_capture = mem_samples[0]
|
||||||
|
per_graph = max(mem_samples[1], _MIN_PER_GRAPH_BYTES) if len(mem_samples) > 1 else 0
|
||||||
|
return first_capture + (total_graphs - 1) * per_graph
|
||||||
|
|
||||||
|
|
||||||
|
def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
|
||||||
|
"""Allocate the smallest KV cache that still lets every graph be captured."""
|
||||||
|
from vllm.v1.core.kv_cache_utils import (
|
||||||
|
get_kv_cache_config_from_groups,
|
||||||
|
get_kv_cache_groups,
|
||||||
|
)
|
||||||
|
|
||||||
|
kv_cache_spec = runner.get_kv_cache_spec()
|
||||||
|
kv_cache_groups = get_kv_cache_groups(runner.vllm_config, kv_cache_spec)
|
||||||
|
# At least one block per sequence is required to capture the graphs.
|
||||||
|
min_blocks = (
|
||||||
|
min(runner.max_num_reqs, runner.compilation_config.max_cudagraph_capture_size)
|
||||||
|
or 1
|
||||||
|
)
|
||||||
|
saved_override = runner.cache_config.num_gpu_blocks_override
|
||||||
|
runner.cache_config.num_gpu_blocks_override = min_blocks
|
||||||
|
try:
|
||||||
|
minimal_config = get_kv_cache_config_from_groups(
|
||||||
|
runner.vllm_config, kv_cache_groups, available_memory=0
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
runner.cache_config.num_gpu_blocks_override = saved_override
|
||||||
|
|
||||||
|
runner.initialize_kv_cache(minimal_config, is_profiling=True)
|
||||||
|
runner.cache_config.num_gpu_blocks = minimal_config.num_blocks
|
||||||
|
|
||||||
|
|
||||||
|
def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
|
||||||
|
"""Release the profiling KV cache and captured graphs while keeping model
|
||||||
|
weights, so the real ``initialize_kv_cache`` starts from a clean slate."""
|
||||||
|
torch.accelerator.synchronize()
|
||||||
|
if hasattr(runner, "kv_caches"):
|
||||||
|
runner.kv_caches.clear()
|
||||||
|
if hasattr(runner, "attn_groups"):
|
||||||
|
runner.attn_groups.clear()
|
||||||
|
if hasattr(runner, "kv_cache_config"):
|
||||||
|
del runner.kv_cache_config
|
||||||
|
# Dropping the manager releases the profiling graphs and throwaway pool.
|
||||||
|
runner.cudagraph_manager = None
|
||||||
|
# Release encoder graphs captured during profiling; the real
|
||||||
|
# capture_model() re-captures them.
|
||||||
|
if runner.model_state.supports_mm_inputs:
|
||||||
|
runner.model_state.encoder_runner.clear()
|
||||||
|
# Detach profiling KV tensors held by attention layers.
|
||||||
|
for layer in runner.compilation_config.static_forward_context.values():
|
||||||
|
if hasattr(layer, "kv_cache"):
|
||||||
|
kv_cache = layer.kv_cache
|
||||||
|
layer.kv_cache = (
|
||||||
|
torch.tensor([]) if isinstance(kv_cache, torch.Tensor) else []
|
||||||
|
)
|
||||||
|
runner.cache_config.num_gpu_blocks = None
|
||||||
|
runner.maybe_remove_all_loras(runner.lora_config)
|
||||||
|
gc.collect()
|
||||||
|
torch.accelerator.empty_cache()
|
||||||
|
|||||||
@@ -94,6 +94,9 @@ from vllm.v1.worker.gpu.cudagraph_utils import (
|
|||||||
BatchExecutionDescriptor,
|
BatchExecutionDescriptor,
|
||||||
ModelCudaGraphManager,
|
ModelCudaGraphManager,
|
||||||
)
|
)
|
||||||
|
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||||
|
profile_cudagraph_memory as _profile_cudagraph_memory,
|
||||||
|
)
|
||||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||||
from vllm.v1.worker.gpu.ec_connector import get_ec_connector
|
from vllm.v1.worker.gpu.ec_connector import get_ec_connector
|
||||||
from vllm.v1.worker.gpu.eplb_utils import EPLBController, step_eplb_after
|
from vllm.v1.worker.gpu.eplb_utils import EPLBController, step_eplb_after
|
||||||
@@ -499,7 +502,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
def get_kv_cache_spec(self):
|
def get_kv_cache_spec(self):
|
||||||
return get_kv_cache_spec(self.vllm_config)
|
return get_kv_cache_spec(self.vllm_config)
|
||||||
|
|
||||||
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
|
def initialize_kv_cache(
|
||||||
|
self, kv_cache_config: KVCacheConfig, is_profiling: bool = False
|
||||||
|
) -> None:
|
||||||
kv_cache_config = deepcopy(kv_cache_config)
|
kv_cache_config = deepcopy(kv_cache_config)
|
||||||
self.kv_cache_config = kv_cache_config
|
self.kv_cache_config = kv_cache_config
|
||||||
|
|
||||||
@@ -585,6 +590,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
tensor_parallel_size=self.parallel_config.tensor_parallel_size,
|
tensor_parallel_size=self.parallel_config.tensor_parallel_size,
|
||||||
kv_cache_config=self.kv_cache_config,
|
kv_cache_config=self.kv_cache_config,
|
||||||
max_num_reqs=self.max_num_reqs,
|
max_num_reqs=self.max_num_reqs,
|
||||||
|
is_profiling=is_profiling,
|
||||||
)
|
)
|
||||||
self.cudagraph_manager = ModelCudaGraphManager(
|
self.cudagraph_manager = ModelCudaGraphManager(
|
||||||
self.vllm_config,
|
self.vllm_config,
|
||||||
@@ -618,7 +624,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
self.kernel_block_sizes,
|
self.kernel_block_sizes,
|
||||||
self.vllm_config,
|
self.vllm_config,
|
||||||
)
|
)
|
||||||
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict)
|
if is_profiling:
|
||||||
|
self.kv_connector = NO_OP_KV_CONNECTOR
|
||||||
|
else:
|
||||||
|
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict)
|
||||||
|
|
||||||
def _init_kv_zero_meta(self) -> None:
|
def _init_kv_zero_meta(self) -> None:
|
||||||
"""Build KV-block zeroing metadata; invoked from gpu_worker."""
|
"""Build KV-block zeroing metadata; invoked from gpu_worker."""
|
||||||
@@ -832,9 +841,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
if self.pooling_runner is not None:
|
if self.pooling_runner is not None:
|
||||||
self.pooling_runner.clear()
|
self.pooling_runner.clear()
|
||||||
|
|
||||||
|
@torch.inference_mode()
|
||||||
def profile_cudagraph_memory(self) -> int:
|
def profile_cudagraph_memory(self) -> int:
|
||||||
# NOTE(woosuk): It is TBD whether we keep this API or not.
|
"""Estimate the GPU memory required to capture CUDA graphs."""
|
||||||
return 0
|
return _profile_cudagraph_memory(self)
|
||||||
|
|
||||||
@torch.inference_mode()
|
@torch.inference_mode()
|
||||||
def capture_model(self) -> int:
|
def capture_model(self) -> int:
|
||||||
@@ -1521,12 +1531,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
slot_mappings, self.kv_cache_config
|
slot_mappings, self.kv_cache_config
|
||||||
)
|
)
|
||||||
assert block_tables is not None
|
assert block_tables is not None
|
||||||
|
attn_groups = self.attn_groups
|
||||||
|
if dummy_run and is_profile:
|
||||||
|
# Mamba layers take a cheap warmup path with no metadata;
|
||||||
|
# attention metadata is still built so those kernels tune.
|
||||||
|
attn_groups = [
|
||||||
|
[g for g in groups if not isinstance(g.kv_cache_spec, MambaSpec)]
|
||||||
|
for groups in attn_groups
|
||||||
|
]
|
||||||
attn_metadata = self.model_state.prepare_attn(
|
attn_metadata = self.model_state.prepare_attn(
|
||||||
input_batch,
|
input_batch,
|
||||||
batch_desc.cg_mode,
|
batch_desc.cg_mode,
|
||||||
block_tables,
|
block_tables,
|
||||||
slot_mappings,
|
slot_mappings,
|
||||||
self.attn_groups,
|
attn_groups,
|
||||||
self.kv_cache_config,
|
self.kv_cache_config,
|
||||||
# FULL replay reads capture-time metadata buffers. Re-stage them
|
# FULL replay reads capture-time metadata buffers. Re-stage them
|
||||||
# from the zeroed dummy block tables instead of retaining state
|
# from the zeroed dummy block tables instead of retaining state
|
||||||
|
|||||||
Reference in New Issue
Block a user