Revert DSv4 eager workspace reuse (#52836)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
a9f4afb66f
commit
f1178f3a06
@@ -29,7 +29,6 @@ from vllm.models.deepseek_v4.common.ops import (
|
||||
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import MXFP4_BLOCK_SIZE
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.deepseek_v4.eager_scratch import DeepseekV4EagerScratchPool
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
DeepseekSparseSWAMetadata,
|
||||
)
|
||||
@@ -185,7 +184,6 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
prefix: str,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
aux_stream_list: list[torch.cuda.Stream] | None = None,
|
||||
eager_scratch_pool: "DeepseekV4EagerScratchPool | None" = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
@@ -274,7 +272,6 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
)
|
||||
self.indexer_rotary_emb = self.rotary_emb
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
self.eager_scratch_pool = eager_scratch_pool
|
||||
|
||||
self.indexer = None
|
||||
if self.compress_ratio == 4:
|
||||
@@ -296,7 +293,6 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
compress_ratio=self.compress_ratio,
|
||||
prefix=f"{prefix}.indexer",
|
||||
aux_stream=indexer_aux_stream,
|
||||
eager_scratch_pool=eager_scratch_pool,
|
||||
)
|
||||
|
||||
self._prepare_and_attn_fn = self._prepare_and_attn
|
||||
@@ -355,7 +351,6 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
rotate=True,
|
||||
prefix=f"{prefix}.compressor",
|
||||
k_cache_prefix=self.prefix,
|
||||
eager_scratch_pool=eager_scratch_pool,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -637,24 +632,10 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
if cache_dtype == torch.uint8:
|
||||
# fp8_ds_mla UE8M0 paged path. Horizontally fused:
|
||||
# Q side: per-head RMSNorm (no weight) + GPT-J RoPE, zero-filling
|
||||
# the padding head slots.
|
||||
# the padding head slots; the kernel allocates and returns
|
||||
# the padded q tensor.
|
||||
# KV side: GPT-J RoPE + UE8M0 FP8 quant + paged cache insert.
|
||||
swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
|
||||
if self.eager_scratch_pool is not None:
|
||||
q_out = self.eager_scratch_pool.q_out(q.shape[0])
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert_out(
|
||||
q,
|
||||
kv,
|
||||
q_out,
|
||||
swa_kv_cache_2d,
|
||||
swa_metadata.slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
self.padded_heads,
|
||||
self.eps,
|
||||
swa_metadata.block_size,
|
||||
)
|
||||
return q_out
|
||||
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q,
|
||||
kv,
|
||||
@@ -703,13 +684,6 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
)
|
||||
return q_fp8
|
||||
|
||||
def _global_topk_output_buffers(
|
||||
self, topk_indices: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor] | None:
|
||||
if self.compress_ratio != 4 or self.eager_scratch_pool is None:
|
||||
return None
|
||||
return self.eager_scratch_pool.global_topk_outputs(topk_indices)
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return self.backend_cls
|
||||
|
||||
@@ -792,7 +766,6 @@ class DeepseekV4Indexer(nn.Module):
|
||||
compress_ratio: int = 1,
|
||||
prefix: str = "",
|
||||
aux_stream: torch.cuda.Stream | None = None,
|
||||
eager_scratch_pool: "DeepseekV4EagerScratchPool | None" = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.vllm_config = vllm_config
|
||||
@@ -805,7 +778,6 @@ class DeepseekV4Indexer(nn.Module):
|
||||
self.rope_dim = config.qk_rope_head_dim # 64
|
||||
self.q_lora_rank = q_lora_rank # 1536
|
||||
self.compress_ratio = compress_ratio
|
||||
self.eager_scratch_pool = eager_scratch_pool
|
||||
self.use_fp4_kv = dsa_indexer_uses_fp4(vllm_config)
|
||||
logger.info_once(
|
||||
"Using %s indexer cache for Lightning Indexer.",
|
||||
@@ -869,7 +841,6 @@ class DeepseekV4Indexer(nn.Module):
|
||||
prefix=f"{prefix}.compressor",
|
||||
k_cache_prefix=self.k_cache.prefix,
|
||||
use_fp4_cache=self.use_fp4_kv,
|
||||
eager_scratch_pool=eager_scratch_pool,
|
||||
)
|
||||
|
||||
self.indexer_op = SparseAttnIndexer(
|
||||
@@ -933,9 +904,6 @@ class DeepseekV4Indexer(nn.Module):
|
||||
# ReplicatedLinear returns (output, bias); bias is None.
|
||||
q, _ = self.wq_b(qr)
|
||||
q = q.view(-1, self.n_head, self.head_dim)
|
||||
outputs = None
|
||||
if self.eager_scratch_pool is not None and self.use_fp4_kv:
|
||||
outputs = self.eager_scratch_pool.indexer_q_outputs(q.shape[0])
|
||||
return fused_indexer_q_rope_quant(
|
||||
positions,
|
||||
q,
|
||||
@@ -944,7 +912,6 @@ class DeepseekV4Indexer(nn.Module):
|
||||
self.softmax_scale,
|
||||
self.n_head**-0.5,
|
||||
use_fp4=self.use_fp4_kv,
|
||||
output_buffers=outputs,
|
||||
)
|
||||
|
||||
# compressor returns None and writes K to the indexer KV cache; the
|
||||
|
||||
@@ -438,7 +438,6 @@ def compute_global_topk_indices_and_lens(
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
is_valid_token: torch.Tensor,
|
||||
output_buffers: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Map local topk indices to global KV cache slots and count valid entries.
|
||||
|
||||
@@ -448,15 +447,8 @@ def compute_global_topk_indices_and_lens(
|
||||
3. Masking padding tokens to length 0
|
||||
"""
|
||||
num_tokens = topk_indices.shape[0]
|
||||
if output_buffers is None:
|
||||
global_topk_indices = torch.empty_like(topk_indices)
|
||||
topk_lens = torch.empty(
|
||||
num_tokens, dtype=torch.int32, device=topk_indices.device
|
||||
)
|
||||
else:
|
||||
global_topk_indices, topk_lens = output_buffers
|
||||
assert global_topk_indices.shape == topk_indices.shape
|
||||
assert topk_lens.shape == (num_tokens,)
|
||||
global_topk_indices = torch.empty_like(topk_indices)
|
||||
topk_lens = torch.empty(num_tokens, dtype=torch.int32, device=topk_indices.device)
|
||||
_compute_global_topk_indices_and_lens_kernel[(num_tokens,)](
|
||||
global_topk_indices,
|
||||
global_topk_indices.stride(0),
|
||||
|
||||
@@ -295,7 +295,6 @@ def fused_indexer_q_rope_quant(
|
||||
index_weights_softmax_scale: float,
|
||||
index_weights_head_scale: float,
|
||||
use_fp4: bool = False,
|
||||
output_buffers: tuple[torch.Tensor, ...] | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
torch.Tensor,
|
||||
@@ -333,13 +332,7 @@ def fused_indexer_q_rope_quant(
|
||||
num_index_q_heads = index_q.shape[1]
|
||||
index_q_head_dim = index_q.shape[2]
|
||||
|
||||
if output_buffers is None:
|
||||
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
|
||||
else:
|
||||
expected_num_buffers = 3 if use_fp4 else 2
|
||||
assert len(output_buffers) == expected_num_buffers
|
||||
index_weights_out = output_buffers[-1]
|
||||
assert index_weights_out.shape == index_weights.shape
|
||||
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
|
||||
|
||||
if use_fp4:
|
||||
assert index_q_head_dim % MXFP4_BLOCK_SIZE == 0, (
|
||||
@@ -347,23 +340,16 @@ def fused_indexer_q_rope_quant(
|
||||
f"size {MXFP4_BLOCK_SIZE}"
|
||||
)
|
||||
num_scale_blocks = index_q_head_dim // MXFP4_BLOCK_SIZE
|
||||
packed_shape = (num_tokens, num_index_q_heads, index_q_head_dim // 2)
|
||||
scale_shape = (num_tokens, num_index_q_heads, num_scale_blocks)
|
||||
if output_buffers is None:
|
||||
index_q_packed = torch.empty(
|
||||
packed_shape,
|
||||
dtype=torch.uint8,
|
||||
device=index_q.device,
|
||||
)
|
||||
index_q_scale = torch.empty(
|
||||
scale_shape,
|
||||
dtype=torch.uint8,
|
||||
device=index_q.device,
|
||||
)
|
||||
else:
|
||||
index_q_packed, index_q_scale, _ = output_buffers
|
||||
assert index_q_packed.shape == packed_shape
|
||||
assert index_q_scale.shape == scale_shape
|
||||
index_q_packed = torch.empty(
|
||||
(num_tokens, num_index_q_heads, index_q_head_dim // 2),
|
||||
dtype=torch.uint8,
|
||||
device=index_q.device,
|
||||
)
|
||||
index_q_scale = torch.empty(
|
||||
(num_tokens, num_index_q_heads, num_scale_blocks),
|
||||
dtype=torch.uint8,
|
||||
device=index_q.device,
|
||||
)
|
||||
if has_cutedsl():
|
||||
# lazily import, otherwise some tests fail due to CUDA driver init failure.
|
||||
from vllm.models.deepseek_v4.nvidia.ops.fused_indexer_q_cutedsl import (
|
||||
@@ -432,11 +418,7 @@ def fused_indexer_q_rope_quant(
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
use_fnuz = fp8_dtype == torch.float8_e4m3fnuz
|
||||
fp8_max = 224.0 if use_fnuz else 448.0
|
||||
if output_buffers is None:
|
||||
index_q_fp8 = torch.empty_like(index_q, dtype=fp8_dtype)
|
||||
else:
|
||||
index_q_fp8, _ = output_buffers
|
||||
assert index_q_fp8.shape == index_q.shape
|
||||
index_q_fp8 = torch.empty_like(index_q, dtype=fp8_dtype)
|
||||
if has_cutedsl():
|
||||
# lazily import, otherwise some tests fail due to CUDA driver init failure.
|
||||
from vllm.models.deepseek_v4.nvidia.ops.fused_indexer_q_cutedsl import (
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, cast
|
||||
from typing import Any, ClassVar, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -35,9 +35,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
SlidingWindowMLASpec,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.deepseek_v4.eager_scratch import DeepseekV4EagerScratchPool
|
||||
|
||||
|
||||
def _prefer_two_stage_compressor() -> bool:
|
||||
# Platforms that favor the triton variant of two-stage compressor split.
|
||||
@@ -229,7 +226,6 @@ class DeepseekCompressor(nn.Module):
|
||||
prefix: str = "",
|
||||
k_cache_prefix="",
|
||||
use_fp4_cache: bool = False,
|
||||
eager_scratch_pool: "DeepseekV4EagerScratchPool | None" = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.compress_ratio = compress_ratio
|
||||
@@ -239,7 +235,6 @@ class DeepseekCompressor(nn.Module):
|
||||
self.prefix = prefix
|
||||
self.k_cache_prefix = k_cache_prefix
|
||||
self.use_fp4_cache = use_fp4_cache
|
||||
self.eager_scratch_pool = eager_scratch_pool
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.rope_head_dim = config.qk_rope_head_dim
|
||||
@@ -433,10 +428,6 @@ class DeepseekCompressor(nn.Module):
|
||||
store_full_fp8=store_full_fp8,
|
||||
fp8_scale=fp8_scale,
|
||||
)
|
||||
if not self.overlap and self.eager_scratch_pool is not None:
|
||||
extra_kwargs["compress_scratch"] = (
|
||||
self.eager_scratch_pool.compressor_scratch(num_actual)
|
||||
)
|
||||
elif self._use_two_stage_fused_compressor:
|
||||
# head=512 cr>=128 (no overlap): two-pass split compressor on the
|
||||
# prefill suffix, single-pass on the decode prefix.
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from math import prod
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import MXFP4_BLOCK_SIZE
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
|
||||
class DeepseekV4EagerScratchPool:
|
||||
"""Model-wide outputs and scratch used inside the attention eager break."""
|
||||
|
||||
_ALIGNMENT = 256
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_num_tokens: int,
|
||||
padded_q_heads: int,
|
||||
q_head_dim: int,
|
||||
index_q_heads: int,
|
||||
index_q_head_dim: int,
|
||||
index_topk: int,
|
||||
device: torch.device | str,
|
||||
) -> None:
|
||||
self.max_num_tokens = max_num_tokens
|
||||
self.index_topk = index_topk
|
||||
self._q = torch.empty(
|
||||
(max_num_tokens, padded_q_heads, q_head_dim),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
fp4_specs = (
|
||||
((max_num_tokens, index_q_heads, index_q_head_dim // 2), torch.uint8),
|
||||
(
|
||||
(
|
||||
max_num_tokens,
|
||||
index_q_heads,
|
||||
index_q_head_dim // MXFP4_BLOCK_SIZE,
|
||||
),
|
||||
torch.uint8,
|
||||
),
|
||||
((max_num_tokens, index_q_heads), torch.float32),
|
||||
)
|
||||
global_specs = (
|
||||
((max_num_tokens, index_topk), torch.int32),
|
||||
((max_num_tokens,), torch.int32),
|
||||
)
|
||||
compressor_specs = (((max_num_tokens, q_head_dim), torch.float32),)
|
||||
# FP4 indexer is C4 only, global mapping after FP4 indexer
|
||||
# compressor scratch is C128 only
|
||||
# so here we use max instead of sum
|
||||
aux_bytes = max(
|
||||
self._packed_size(specs)
|
||||
for specs in (fp4_specs, global_specs, compressor_specs)
|
||||
)
|
||||
storage = torch.empty(aux_bytes, dtype=torch.uint8, device=device)
|
||||
|
||||
self._q_outputs: dict[int, torch.Tensor] = {}
|
||||
fp4_values, fp4_scales, fp4_weights = self._views(storage, fp4_specs)
|
||||
self._fp4_template = (fp4_values, fp4_scales, fp4_weights)
|
||||
self._fp4_outputs: dict[
|
||||
int, tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
] = {}
|
||||
global_indices, global_lens = self._views(storage, global_specs)
|
||||
self._global_template = (global_indices, global_lens)
|
||||
self._global_outputs: dict[int, tuple[torch.Tensor, torch.Tensor]] = {}
|
||||
self._compressor_template = self._views(storage, compressor_specs)[0]
|
||||
self._compressor_outputs: dict[int, torch.Tensor] = {}
|
||||
self._storage = storage
|
||||
|
||||
@classmethod
|
||||
def _packed_size(
|
||||
cls, specs: tuple[tuple[tuple[int, ...], torch.dtype], ...]
|
||||
) -> int:
|
||||
offset = 0
|
||||
for shape, dtype in specs:
|
||||
offset = round_up(offset, cls._ALIGNMENT) + prod(shape) * dtype.itemsize
|
||||
return round_up(offset, cls._ALIGNMENT)
|
||||
|
||||
@classmethod
|
||||
def _views(
|
||||
cls,
|
||||
storage: torch.Tensor,
|
||||
specs: tuple[tuple[tuple[int, ...], torch.dtype], ...],
|
||||
) -> list[torch.Tensor]:
|
||||
offset = 0
|
||||
views = []
|
||||
for shape, dtype in specs:
|
||||
offset = round_up(offset, cls._ALIGNMENT)
|
||||
num_bytes = prod(shape) * dtype.itemsize
|
||||
views.append(storage[offset : offset + num_bytes].view(dtype).view(shape))
|
||||
offset += num_bytes
|
||||
return views
|
||||
|
||||
def q_out(self, num_tokens: int) -> torch.Tensor:
|
||||
output = self._q_outputs.get(num_tokens)
|
||||
if output is None:
|
||||
output = self._q[:num_tokens]
|
||||
self._q_outputs[num_tokens] = output
|
||||
return output
|
||||
|
||||
def compressor_scratch(self, num_tokens: int) -> torch.Tensor:
|
||||
output = self._compressor_outputs.get(num_tokens)
|
||||
if output is None:
|
||||
output = self._compressor_template[:num_tokens]
|
||||
self._compressor_outputs[num_tokens] = output
|
||||
return output
|
||||
|
||||
def indexer_q_outputs(
|
||||
self,
|
||||
num_tokens: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
output = self._fp4_outputs.get(num_tokens)
|
||||
if output is None:
|
||||
values, scales, weights = self._fp4_template
|
||||
output = (
|
||||
values[:num_tokens],
|
||||
scales[:num_tokens],
|
||||
weights[:num_tokens],
|
||||
)
|
||||
self._fp4_outputs[num_tokens] = output
|
||||
return output
|
||||
|
||||
def global_topk_outputs(
|
||||
self, topk_indices: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_tokens, topk = topk_indices.shape
|
||||
assert topk == self.index_topk
|
||||
output = self._global_outputs.get(num_tokens)
|
||||
if output is None:
|
||||
indices, lens = self._global_template
|
||||
output = (indices[:num_tokens], lens[:num_tokens])
|
||||
self._global_outputs[num_tokens] = output
|
||||
return output
|
||||
@@ -763,9 +763,6 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
|
||||
attn_metadata.block_table[:num_decodes],
|
||||
block_size,
|
||||
is_valid,
|
||||
output_buffers=self._global_topk_output_buffers(
|
||||
self.topk_indices_buffer[:num_decode_tokens]
|
||||
),
|
||||
)
|
||||
)
|
||||
extra_sparse_indices = global_indices.view(num_decode_tokens, 1, -1)
|
||||
@@ -855,7 +852,6 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
|
||||
attn_metadata.block_table,
|
||||
block_size,
|
||||
swa_metadata.is_valid_token[prefill_token_slice],
|
||||
output_buffers=self._global_topk_output_buffers(local_topk_indices),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -197,9 +197,6 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
|
||||
attn_metadata.block_table[:num_decodes],
|
||||
block_size,
|
||||
is_valid,
|
||||
output_buffers=self._global_topk_output_buffers(
|
||||
self.topk_indices_buffer[:num_decode_tokens]
|
||||
),
|
||||
)
|
||||
topk_indices = global_indices.view(num_decode_tokens, 1, -1)
|
||||
else:
|
||||
|
||||
@@ -72,7 +72,6 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.eager_scratch import DeepseekV4EagerScratchPool
|
||||
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
|
||||
DeepseekV4FlashInferMLAAttention,
|
||||
DeepseekV4FlashInferSM120Attention,
|
||||
@@ -815,7 +814,6 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
prefix,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
aux_stream_list: list[torch.cuda.Stream] | None = None,
|
||||
eager_scratch_pool: DeepseekV4EagerScratchPool | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -829,7 +827,6 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.attn",
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
aux_stream_list=aux_stream_list,
|
||||
eager_scratch_pool=eager_scratch_pool,
|
||||
)
|
||||
if self.use_sequence_parallel:
|
||||
self.attn.wo_b.reduce_results = False
|
||||
@@ -1017,22 +1014,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
# (compressor kv_score, indexer.weights_proj, indexer.compressor
|
||||
# kv_score). fused_wqa_wkv stays on the default stream.
|
||||
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
|
||||
padded_heads = _select_dsv4_attn_cls(vllm_config).get_padded_num_q_heads(
|
||||
config.num_attention_heads // get_tensor_model_parallel_world_size()
|
||||
)
|
||||
self.eager_scratch_pool: DeepseekV4EagerScratchPool | None = None
|
||||
if not vllm_config.parallel_config.use_ubatching:
|
||||
# TODO: support dbo if needed
|
||||
# this requires the buffer to have ubatch dim
|
||||
self.eager_scratch_pool = DeepseekV4EagerScratchPool(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
padded_heads,
|
||||
config.head_dim,
|
||||
config.index_n_heads,
|
||||
config.index_head_dim,
|
||||
config.index_topk,
|
||||
current_platform.device_type,
|
||||
)
|
||||
|
||||
# Reserved topk indices buffer for all Indexer layers to reuse.
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
@@ -1058,7 +1039,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
aux_stream_list=aux_stream_list,
|
||||
eager_scratch_pool=self.eager_scratch_pool,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
@@ -2097,7 +2097,6 @@ def compress_norm_rope_store_cutedsl(
|
||||
store_full_kv: bool = False,
|
||||
store_full_fp8: bool = False,
|
||||
fp8_scale: torch.Tensor | None = None,
|
||||
compress_scratch: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
if compress_ratio == 4:
|
||||
# For C4A, the single fused kernel is faster than the two-kernel version.
|
||||
@@ -2130,15 +2129,11 @@ def compress_norm_rope_store_cutedsl(
|
||||
)
|
||||
else:
|
||||
# For C128, the two-kernel version is faster than the single fused kernel.
|
||||
if compress_scratch is None:
|
||||
compressed_kv = torch.empty(
|
||||
(num_actual, head_dim),
|
||||
dtype=torch.float32,
|
||||
device=state_cache.device,
|
||||
)
|
||||
else:
|
||||
assert compress_scratch.shape == (num_actual, head_dim)
|
||||
compressed_kv = compress_scratch
|
||||
compressed_kv = torch.empty(
|
||||
(num_actual, head_dim),
|
||||
dtype=torch.float32,
|
||||
device=state_cache.device,
|
||||
)
|
||||
split_kv_compress_norm_rope_insert_sparse_attn_cutedsl(
|
||||
state_cache,
|
||||
token_to_req_indices,
|
||||
|
||||
Reference in New Issue
Block a user