[Perf][DSv4.1] Run decoder replay layers in CUDA graphs and trim in PIECEWISE graphs (#59532)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
21d93d0d8c
commit
eb2901b735
@@ -17,7 +17,8 @@ from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.core.sched.output import NewRequestData
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
@@ -126,6 +127,16 @@ class ModelState(ABC):
|
||||
def apply_staged_writes(self) -> None:
|
||||
return None
|
||||
|
||||
def capture_inner_cudagraphs(
|
||||
self,
|
||||
input_buffers: InputBuffers,
|
||||
block_tables: BlockTables,
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
"""Capture the CUDA graphs this state runs inside the model's forward."""
|
||||
return None
|
||||
|
||||
def get_additional_cg_support(self) -> tuple[AttentionCGSupport, str | None]:
|
||||
"""Cudagraph support of attention groups this ModelState builds outside
|
||||
``init_attn_backend`` (e.g. encoder-only layers).
|
||||
|
||||
Reference in New Issue
Block a user