Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
@@ -56,10 +56,7 @@ class SpeculatorCudaGraphManager(CudaGraphManager):
|
||||
block_tables,
|
||||
attn_groups,
|
||||
kv_cache_config,
|
||||
skip_attn=(
|
||||
desc.cg_mode == CUDAGraphMode.PIECEWISE
|
||||
and not self.use_breakable_cg
|
||||
),
|
||||
full_cudagraph=desc.cg_mode == CUDAGraphMode.FULL,
|
||||
)
|
||||
|
||||
return lambda cg_mode: forward_fn(
|
||||
|
||||
Reference in New Issue
Block a user