[Bugfix][Spec Decode] Cap DFlash/DSpark profiling query batch (#56448)
Signed-off-by: wangyicong <wangyicong@bytedance.com>
This commit is contained in:
@@ -57,6 +57,8 @@ def _target_feeds_hc_residual(vllm_config: VllmConfig) -> bool:
|
||||
|
||||
|
||||
class BaseSpeculator(ABC):
|
||||
num_query_per_req: int = 1
|
||||
|
||||
@abstractmethod
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user