[Perf][DSv4.1] Run decoder replay layers in CUDA graphs and trim in PIECEWISE graphs (#59532)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Juntian Liu
2026-10-05 22:11:45 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 21d93d0d8c
commit eb2901b735
10 changed files with 445 additions and 71 deletions
+6
View File
@@ -1077,6 +1077,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.lora_config, self
),
)
self.model_state.capture_inner_cudagraphs(
input_buffers,
self.block_tables,
self.attn_groups,
self.kv_cache_config,
)
if self.speculator is not None:
with use_workspace_lane(self._draft_workspace_lane):
self.speculator.capture()
+12 -1
View File
@@ -17,7 +17,8 @@ from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
from vllm.v1.worker.gpu.states import RequestState
@@ -126,6 +127,16 @@ class ModelState(ABC):
def apply_staged_writes(self) -> None:
return None
def capture_inner_cudagraphs(
self,
input_buffers: InputBuffers,
block_tables: BlockTables,
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
) -> None:
"""Capture the CUDA graphs this state runs inside the model's forward."""
return None
def get_additional_cg_support(self) -> tuple[AttentionCGSupport, str | None]:
"""Cudagraph support of attention groups this ModelState builds outside
``init_attn_backend`` (e.g. encoder-only layers).