[Bugfix] DFlash FP8 KV-Cache (#42692)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
@@ -116,7 +116,7 @@ class SpecDecodeBaseProposer:
|
||||
|
||||
self.max_batch_size = vllm_config.scheduler_config.max_num_seqs
|
||||
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
self.token_arange_np = np.arange(self.max_num_tokens)
|
||||
self.token_arange_np = np.arange(self.max_num_tokens, dtype=np.int32)
|
||||
|
||||
# Can be specialized by methods like DFlash to reduce the limit
|
||||
self.max_query_tokens = self.max_num_tokens
|
||||
|
||||
Reference in New Issue
Block a user