[Bugfix] DFlash FP8 KV-Cache (#42692)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
@@ -247,8 +247,10 @@ class DFlashQwen3Model(nn.Module):
|
|||||||
[
|
[
|
||||||
DFlashQwen3DecoderLayer(
|
DFlashQwen3DecoderLayer(
|
||||||
current_vllm_config,
|
current_vllm_config,
|
||||||
prefix=maybe_prefix(prefix, f"layers.{layer_idx + start_layer_id}"),
|
|
||||||
config=self.config,
|
config=self.config,
|
||||||
|
cache_config=current_vllm_config.cache_config,
|
||||||
|
quant_config=self.quant_config,
|
||||||
|
prefix=maybe_prefix(prefix, f"layers.{layer_idx + start_layer_id}"),
|
||||||
)
|
)
|
||||||
for layer_idx in range(self.config.num_hidden_layers)
|
for layer_idx in range(self.config.num_hidden_layers)
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -116,7 +116,7 @@ class SpecDecodeBaseProposer:
|
|||||||
|
|
||||||
self.max_batch_size = vllm_config.scheduler_config.max_num_seqs
|
self.max_batch_size = vllm_config.scheduler_config.max_num_seqs
|
||||||
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||||
self.token_arange_np = np.arange(self.max_num_tokens)
|
self.token_arange_np = np.arange(self.max_num_tokens, dtype=np.int32)
|
||||||
|
|
||||||
# Can be specialized by methods like DFlash to reduce the limit
|
# Can be specialized by methods like DFlash to reduce the limit
|
||||||
self.max_query_tokens = self.max_num_tokens
|
self.max_query_tokens = self.max_num_tokens
|
||||||
|
|||||||
Reference in New Issue
Block a user