Add Laguna XS.2.1 DFlash drafter support (#46853)

Signed-off-by: Adam Baranowski <adam.baranowski@poolside.ai>
This commit is contained in:
adamkbaranowski
2026-07-02 18:09:27 -07:00
committed by GitHub
parent 442ccc6098
commit 4c3c64fcf7
7 changed files with 494 additions and 47 deletions
+64 -41
View File
@@ -410,18 +410,11 @@ class DFlashQwen3Model(nn.Module):
embeds = torch.where(is_mask, self.mask_embedding.to(embeds.dtype), embeds)
return embeds
def _build_fused_kv_buffers(self) -> None:
"""Build fused weight buffers for precompute_and_store_context_kv.
Must be called after weights are loaded. Stacks the KV-projection
weights, K-norm weights, and RoPE parameters from every attention
layer so that precompute_and_store_context_kv can run one fused
GEMM for all layers at once. Also aliases the weight of the hidden_norm.
"""
layers_attn = [layer.self_attn for layer in self.layers]
attn0 = layers_attn[0]
has_bias = attn0.qkv_proj.bias is not None
def _build_context_kv_buffers(
self,
layers_attn: list[nn.Module],
has_bias: bool,
) -> None:
self._hidden_norm_weight = self.hidden_norm.weight.data
# KV projection weights: [num_layers * 2 * kv_size, hidden_size]
@@ -439,6 +432,20 @@ class DFlashQwen3Model(nn.Module):
[a.k_norm.weight.data for a in layers_attn], dim=0
).contiguous()
def _build_fused_kv_buffers(self) -> None:
"""Build fused weight buffers for precompute_and_store_context_kv.
Must be called after weights are loaded. Stacks the KV-projection
weights, K-norm weights, and RoPE parameters from every attention
layer so that precompute_and_store_context_kv can run one fused
GEMM for all layers at once. Also aliases the weight of the hidden_norm.
"""
layers_attn = [layer.self_attn for layer in self.layers]
attn0 = layers_attn[0]
has_bias = attn0.qkv_proj.bias is not None
self._build_context_kv_buffers(layers_attn, has_bias)
# RoPE parameters
self._rope_head_size = attn0.rotary_emb.head_size
self._rope_cos_sin_cache = attn0.rotary_emb.cos_sin_cache
@@ -468,6 +475,49 @@ class DFlashQwen3Model(nn.Module):
# References to inner Attention layers for direct cache writes
self._attn_layers = [layer.self_attn.attn for layer in self.layers]
def _project_context_kv(
self,
context_states: torch.Tensor,
num_ctx: int,
num_layers: int,
num_kv_heads: int,
head_dim: int,
) -> tuple[torch.Tensor, torch.Tensor]:
# --- Fused KV projection (one GEMM for all layers) ---
normed_context_states = torch.empty_like(context_states)
ops.rms_norm(
normed_context_states,
context_states,
self._hidden_norm_weight,
self._rms_norm_eps,
)
all_kv_flat = F.linear(
normed_context_states, self._fused_kv_weight, self._fused_kv_bias
)
# Single contiguous copy that separates K/V and transposes to
# layer-major layout. Result: [2, L, num_ctx, nkv, hd] contiguous.
# Indexing dim-0 gives contiguous [L, num_ctx, nkv, hd] for K and V.
all_kv = (
all_kv_flat.view(num_ctx, num_layers, 2, num_kv_heads, head_dim)
.permute(2, 1, 0, 3, 4)
.contiguous()
)
all_k = all_kv[0] # [L, num_ctx, nkv, hd], contiguous
all_v = all_kv[1] # [L, num_ctx, nkv, hd], contiguous
return all_k, all_v
def _normalize_context_k(self, all_k: torch.Tensor) -> torch.Tensor:
# --- Grouped RMSNorm K across all layers ([L, num_ctx, nkv, hd]) ---
# The weight is selected per layer by the outermost (layer) index.
all_k_normed = torch.empty_like(all_k)
ops.rms_norm(
all_k_normed,
all_k,
self._k_norm_weights,
self._rms_norm_eps,
)
return all_k_normed
def precompute_and_store_context_kv(
self,
context_states: torch.Tensor,
@@ -499,35 +549,8 @@ class DFlashQwen3Model(nn.Module):
hd = self._head_dim
nkv = self._num_kv_heads
# --- Fused KV projection (one GEMM for all layers) ---
normed_context_states = torch.empty_like(context_states)
ops.rms_norm(
normed_context_states,
context_states,
self._hidden_norm_weight,
self._rms_norm_eps,
)
all_kv_flat = F.linear(
normed_context_states, self._fused_kv_weight, self._fused_kv_bias
)
# Single contiguous copy that separates K/V and transposes to
# layer-major layout. Result: [2, L, num_ctx, nkv, hd] contiguous.
# Indexing dim-0 gives contiguous [L, num_ctx, nkv, hd] for K and V.
all_kv = (
all_kv_flat.view(num_ctx, L, 2, nkv, hd).permute(2, 1, 0, 3, 4).contiguous()
)
all_k = all_kv[0] # [L, num_ctx, nkv, hd], contiguous
all_v = all_kv[1] # [L, num_ctx, nkv, hd], contiguous
# --- Grouped RMSNorm K across all layers ([L, num_ctx, nkv, hd]) ---
# The weight is selected per layer by the outermost (layer) index.
all_k_normed = torch.empty_like(all_k)
ops.rms_norm(
all_k_normed,
all_k,
self._k_norm_weights,
self._rms_norm_eps,
)
all_k, all_v = self._project_context_kv(context_states, num_ctx, L, nkv, hd)
all_k_normed = self._normalize_context_k(all_k)
# --- Fused RoPE across all layers ---
# View as [L * num_ctx, kv] so RoPE sees one big batch (no copy).