[Model] Add PW CUDA graph support for Inkling [2/N] (#48822)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Bugen Zhao
Giancarlo Delfin
Isotr0py
Isotr0py
Jee Jee Li
Roger Wang
Yifan Qiao
Claude Fable 5
OpenAI Codex
parent
ce65385618
commit
251f7e478e
@@ -5,6 +5,7 @@ from typing import Any
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import is_breakable_cudagraph_enabled
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.core.sched.output import NewRequestData
|
||||
@@ -139,7 +140,10 @@ class DefaultModelState(ModelState):
|
||||
kv_cache_config: KVCacheConfig,
|
||||
for_capture: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
if cudagraph_mode == CUDAGraphMode.FULL:
|
||||
if cudagraph_mode == CUDAGraphMode.FULL or (
|
||||
cudagraph_mode == CUDAGraphMode.PIECEWISE
|
||||
and is_breakable_cudagraph_enabled()
|
||||
):
|
||||
# Use padded sizes - padding is handled by model_runner.prepare_attn.
|
||||
num_reqs = input_batch.num_reqs_after_padding
|
||||
num_tokens = input_batch.num_tokens_after_padding
|
||||
|
||||
Reference in New Issue
Block a user