[Kernel] Reuse Qwen4Exp HC combine-norm for MTP input (#54687)
Signed-off-by: Canlin Guo <canlinguosdu@gmail.com> Signed-off-by: Canlin <canlinguosdu@gmail.com>
This commit is contained in:
@@ -68,6 +68,18 @@ def test_hc_combine() -> None:
|
||||
torch.testing.assert_close(actual, expected.flatten(-2).to(torch.bfloat16))
|
||||
|
||||
|
||||
def test_hc_combine_unit_injection() -> None:
|
||||
torch.manual_seed(0)
|
||||
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
residual = torch.randn(2, HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual = hc_combine(residual, block_output, None, HC)
|
||||
expected = residual.unflatten(-1, (HC, HIDDEN_SIZE))
|
||||
expected = expected + block_output.unsqueeze(-2)
|
||||
|
||||
assert torch.equal(actual, expected.flatten(-2))
|
||||
|
||||
|
||||
def test_hc_combine_norm() -> None:
|
||||
torch.manual_seed(0)
|
||||
block_output = torch.randn(2, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
@@ -92,3 +104,24 @@ def test_hc_combine_norm() -> None:
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
torch.testing.assert_close(actual_norm, expected_norm.to(torch.bfloat16))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17, 2048])
|
||||
def test_hc_combine_norm_unit_injection(num_tokens: int) -> None:
|
||||
torch.manual_seed(0)
|
||||
embedding = torch.randn(
|
||||
num_tokens, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
hidden = torch.randn(
|
||||
num_tokens, HC, HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
weight = torch.randn(HYPER_HIDDEN_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
actual, actual_norm = hc_combine_norm(
|
||||
hidden.flatten(1), embedding, None, weight, EPS, HC
|
||||
)
|
||||
|
||||
expected = (hidden + embedding.unsqueeze(1)).flatten(1)
|
||||
expected_norm = grouped_gemma_rmsnorm(expected, weight, EPS, HC)
|
||||
assert torch.equal(actual, expected)
|
||||
torch.testing.assert_close(actual_norm, expected_norm)
|
||||
|
||||
@@ -52,9 +52,10 @@ class GatedResidual(nn.Module):
|
||||
|
||||
``combine_and_mix()`` runs the pre pipeline (grouped GemmaRMSNorm -> merged
|
||||
low-rank down+inject GEMM -> silu -> up GEMM -> sigmoid -> gated mean
|
||||
over the HC streams). When passed a pending block output and an injection,
|
||||
it fuses their residual combine with the RMSNorm. Final mixers use
|
||||
``use_combine=False`` and do not produce a new injection.
|
||||
over the HC streams). When passed a pending block output, it fuses its
|
||||
residual combine with the RMSNorm. A missing injection selects unit-weight
|
||||
combine. Final mixers use ``use_combine=False`` and do not produce a new
|
||||
injection.
|
||||
|
||||
Weights: the norm owns the grouped GemmaRMSNorm affine; the projections
|
||||
are vLLM Linear modules (merged replicated linear for down+inject), so
|
||||
@@ -153,13 +154,14 @@ class GatedResidual(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
prev_block_output: torch.Tensor,
|
||||
prev_injection: torch.Tensor,
|
||||
prev_injection: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""Consume a pending combine, then prepare the next block input.
|
||||
|
||||
``hidden_states`` is the multi-stream state from before the pending
|
||||
block's mix. Its combine with ``block_output`` is fused with this
|
||||
module's input RMSNorm.
|
||||
module's input RMSNorm. A missing injection applies the block output
|
||||
to every stream with unit weight.
|
||||
"""
|
||||
hidden_states, xn = hc_combine_norm(
|
||||
hidden_states,
|
||||
@@ -189,7 +191,7 @@ class GatedResidual(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection: torch.Tensor,
|
||||
injection: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
return hc_combine(hidden_states, block_output, injection, self.hc_count)
|
||||
|
||||
|
||||
@@ -284,11 +284,13 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
query_start_loc: torch.Tensor | None,
|
||||
ngram_context: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
if prev_block_output is None:
|
||||
assert prev_injection is None
|
||||
attn_hc = self.attn_hyper_connection
|
||||
if self.ple is not None:
|
||||
# PLE adds directly to the multi-stream state, so pending HC state
|
||||
# must be materialized before the addition.
|
||||
if prev_block_output is not None and prev_injection is not None:
|
||||
if prev_block_output is not None:
|
||||
hidden_states = attn_hc.combine(
|
||||
hidden_states, prev_block_output, prev_injection
|
||||
)
|
||||
@@ -304,7 +306,7 @@ class Qwen4ExpDecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# Fuse a pending combine with this HC module's mix when possible.
|
||||
if prev_block_output is not None and prev_injection is not None:
|
||||
if prev_block_output is not None:
|
||||
hidden_states, block_input, injection = attn_hc.combine_and_mix(
|
||||
hidden_states, prev_block_output, prev_injection
|
||||
)
|
||||
|
||||
@@ -280,6 +280,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
|
||||
hc_count = self.hc_count
|
||||
hidden_size = self.hidden_size
|
||||
prev_block_output: torch.Tensor | None = None
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
assert hidden_states is not None
|
||||
@@ -300,10 +301,8 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
num_tokens, hc_count, hidden_size
|
||||
)
|
||||
hidden_states = self.fc_hidden(hidden_states)
|
||||
# Add the embedding residual to every branch, then fold back
|
||||
# to [T, hc_count*H] (HC outer, HS inner) for the HC decoder.
|
||||
hidden_states = inputs_embeds.unsqueeze(-2) + hidden_states
|
||||
hidden_states = hidden_states.flatten(-2)
|
||||
prev_block_output = inputs_embeds
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
@@ -312,7 +311,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module):
|
||||
layer = self.layers[current_step_idx]
|
||||
hidden_states, block_output, injection = layer(
|
||||
hidden_states=hidden_states,
|
||||
prev_block_output=None,
|
||||
prev_block_output=prev_block_output,
|
||||
prev_injection=None,
|
||||
positions=positions,
|
||||
input_ids=None,
|
||||
|
||||
@@ -215,14 +215,17 @@ def _hc_combine_kernel(
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
|
||||
inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
|
||||
if inj_ptr is not None:
|
||||
inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
|
||||
block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)
|
||||
res = tl.load(res_ptr + row * stride_res + offs, mask, other=0.0)
|
||||
|
||||
# Keeping HC as a broadcast dimension is faster here than four separate
|
||||
# residual load/store sequences.
|
||||
inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
|
||||
out = res.to(tl.float32) + block.to(tl.float32)[None, :] * inj[:, None]
|
||||
if inj_ptr is not None:
|
||||
inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
|
||||
block = block.to(tl.float32)[None, :] * inj[:, None]
|
||||
out = res.to(tl.float32) + block.to(tl.float32)
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
@@ -232,17 +235,20 @@ def _hc_combine_kernel(
|
||||
def _hc_combine(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
injection_logits: torch.Tensor | None,
|
||||
hc_count: int,
|
||||
) -> torch.Tensor:
|
||||
N, DIM = residual.shape
|
||||
assert DIM % hc_count == 0
|
||||
hc_dim = DIM // hc_count
|
||||
assert block_output.shape == (N, hc_dim)
|
||||
assert injection_logits.shape == (N, hc_count)
|
||||
assert residual.stride(1) == 1
|
||||
assert block_output.stride(1) == 1
|
||||
assert injection_logits.stride(1) == 1
|
||||
if injection_logits is not None:
|
||||
assert injection_logits.shape == (N, hc_count)
|
||||
assert injection_logits.stride(1) == 1
|
||||
|
||||
stride_injection = injection_logits.stride(0) if injection_logits is not None else 0
|
||||
|
||||
out = residual.new_empty(residual.shape)
|
||||
BLOCK_SIZE = 512
|
||||
@@ -253,7 +259,7 @@ def _hc_combine(
|
||||
out,
|
||||
block_output.stride(0),
|
||||
residual.stride(0),
|
||||
injection_logits.stride(0),
|
||||
stride_injection,
|
||||
out.stride(0),
|
||||
hc_dim,
|
||||
hc_count,
|
||||
@@ -287,8 +293,9 @@ def _hc_combine_norm_kernel(
|
||||
NUM_TILES: tl.constexpr = triton.cdiv(HC_DIM, BLOCK_SIZE)
|
||||
NUM_TILES_PAD: tl.constexpr = triton.next_power_of_2(NUM_TILES)
|
||||
|
||||
row = tl.program_id(0)
|
||||
stream = tl.program_id(1)
|
||||
pid = tl.program_id(0)
|
||||
row = pid // HC
|
||||
stream = pid % HC
|
||||
offs_hc = tl.arange(0, HC_PAD)
|
||||
mask_hc = offs_hc < HC
|
||||
tile_ids = tl.arange(0, NUM_TILES_PAD)
|
||||
@@ -305,27 +312,34 @@ def _hc_combine_norm_kernel(
|
||||
# Start the uncached residual load first, then issue the other combine
|
||||
# loads before consuming any of them.
|
||||
res = tl.load(res_ptr + row * stride_res + offs, mask_inner, other=0.0)
|
||||
inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
|
||||
block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)
|
||||
inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
|
||||
inj = tl.sum(tl.where(offs_hc == stream, inj, 0.0))
|
||||
if inj_ptr is not None:
|
||||
inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
|
||||
block = tl.load(
|
||||
block_ptr + row * stride_block + offs_inner,
|
||||
mask_inner,
|
||||
other=0.0,
|
||||
)
|
||||
if inj_ptr is not None:
|
||||
inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
|
||||
block = block.to(tl.float32) * tl.sum(tl.where(offs_hc == stream, inj, 0.0))
|
||||
# Round the materialized combine result before normalization. This matches
|
||||
# the unfused combine -> RMSNorm boundary.
|
||||
out = (res.to(tl.float32) + block.to(tl.float32) * inj).to(out_ptr.dtype.element_ty)
|
||||
out = (res.to(tl.float32) + block.to(tl.float32)).to(out_ptr.dtype.element_ty)
|
||||
if inj_ptr is None:
|
||||
w = tl.load(w_ptr + w_offs, mask_inner, other=0.0)
|
||||
tl.store(out_ptr + row * stride_out + offs, out, mask=mask_inner)
|
||||
|
||||
out = out.to(tl.float32)
|
||||
# Keep the two-axis reduction: flattening the padded tile is ~40% slower
|
||||
# at decode sizes.
|
||||
sum_sq = tl.sum(tl.sum(out * out, axis=1), axis=0)
|
||||
rrms = tl.rsqrt(sum_sq / HC_DIM + EPS)
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
# Loading the weight earlier helps decode but keeps the tile live across
|
||||
# the reduction and regresses larger batches, so defer it to the norm.
|
||||
w = tl.load(w_ptr + w_offs, mask_inner, other=0.0)
|
||||
if inj_ptr is not None:
|
||||
# Loading the weight earlier helps decode but keeps the tile live across
|
||||
# the reduction and regresses larger batches, so defer it to the norm.
|
||||
w = tl.load(w_ptr + w_offs, mask_inner, other=0.0)
|
||||
y = out * rrms
|
||||
y += y * w.to(tl.float32)
|
||||
tl.store(y_ptr + row * stride_y + offs, y, mask_inner)
|
||||
@@ -334,7 +348,7 @@ def _hc_combine_norm_kernel(
|
||||
def _hc_combine_norm(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
injection_logits: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_count: int,
|
||||
@@ -343,17 +357,20 @@ def _hc_combine_norm(
|
||||
assert DIM % hc_count == 0
|
||||
hc_dim = DIM // hc_count
|
||||
assert block_output.shape == (N, hc_dim)
|
||||
assert injection_logits.shape == (N, hc_count)
|
||||
assert residual.stride(1) == 1
|
||||
assert block_output.stride(1) == 1
|
||||
assert injection_logits.stride(1) == 1
|
||||
if injection_logits is not None:
|
||||
assert injection_logits.shape == (N, hc_count)
|
||||
assert injection_logits.stride(1) == 1
|
||||
assert norm_weight.is_contiguous()
|
||||
assert norm_weight.numel() in (hc_dim, DIM)
|
||||
|
||||
stride_injection = injection_logits.stride(0) if injection_logits is not None else 0
|
||||
|
||||
out = residual.new_empty(residual.shape)
|
||||
y = residual.new_empty(residual.shape)
|
||||
BLOCK_SIZE = 512
|
||||
_hc_combine_norm_kernel[(N, hc_count)](
|
||||
_hc_combine_norm_kernel[(N * hc_count,)](
|
||||
block_output,
|
||||
residual,
|
||||
injection_logits,
|
||||
@@ -362,7 +379,7 @@ def _hc_combine_norm(
|
||||
y,
|
||||
block_output.stride(0),
|
||||
residual.stride(0),
|
||||
injection_logits.stride(0),
|
||||
stride_injection,
|
||||
out.stride(0),
|
||||
y.stride(0),
|
||||
hc_dim,
|
||||
@@ -389,7 +406,7 @@ def _hc_gate_mix_fake(
|
||||
def _hc_combine_fake(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
injection_logits: torch.Tensor | None,
|
||||
hc_count: int,
|
||||
) -> torch.Tensor:
|
||||
del block_output, injection_logits, hc_count
|
||||
@@ -399,7 +416,7 @@ def _hc_combine_fake(
|
||||
def _hc_combine_norm_fake(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
injection_logits: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_count: int,
|
||||
@@ -452,7 +469,7 @@ def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Ten
|
||||
def hc_combine(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
injection_logits: torch.Tensor | None,
|
||||
hc_count: int,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.qwen4_exp_hc_combine(
|
||||
@@ -463,7 +480,7 @@ def hc_combine(
|
||||
def hc_combine_norm(
|
||||
residual: torch.Tensor,
|
||||
block_output: torch.Tensor,
|
||||
injection_logits: torch.Tensor,
|
||||
injection_logits: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_count: int,
|
||||
|
||||
Reference in New Issue
Block a user