[Kernel] Resubmit PR 48666 - Gemma4 FP8 KV FA4 head dim 512 backend selection (#53175)

Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com>
This commit is contained in:
Jhao-Ting Chen
2026-09-23 20:29:06 -07:00
committed by GitHub
parent e78e367c6e
commit 4fb767ff0a
47 changed files with 241 additions and 103 deletions
@@ -24,6 +24,27 @@ from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
logger = init_logger(__name__)
def _copy_target_kv_scales(attn: nn.Module, target_attn: nn.Module) -> None:
"""Copy target KV scales while preserving their tensor representation.
Default attention scales are scalar buffers, while some quantization
methods replace them with length-one or per-head parameters. Preserve the
draft layer's registration type so the shared KV cache is interpreted with
the target's values and shapes without aliasing target parameters.
"""
for scale_name in ("_k_scale", "_v_scale"):
draft_scale = getattr(attn, scale_name)
target_scale = getattr(target_attn, scale_name)
scale = target_scale.detach().clone()
if isinstance(draft_scale, nn.Parameter):
scale = nn.Parameter(scale, requires_grad=False)
setattr(attn, scale_name, scale)
for scale_name in ("_k_scale_float", "_v_scale_float"):
setattr(attn, scale_name, getattr(target_attn, scale_name))
for scale_name in ("_k_scale_cpu", "_v_scale_cpu"):
getattr(attn, scale_name).copy_(getattr(target_attn, scale_name))
class Gemma4Speculator(AutoRegressiveSpeculator):
@property
def advance_draft_positions(self) -> bool:
@@ -77,7 +98,7 @@ class Gemma4Speculator(AutoRegressiveSpeculator):
model: nn.Module,
target_attn_layer_names: set[str],
) -> None:
"""Wire draft layers to share KV with the target model.
"""Wire draft layers to share KV and KV scales with the target model.
Each draft decoder layer is mapped to the last non-KV-shared
target layer of the same attention type (sliding or full).
@@ -92,7 +113,10 @@ class Gemma4Speculator(AutoRegressiveSpeculator):
return
target_num_kv_shared = getattr(target_text_config, "num_kv_shared_layers", 0)
num_non_shared = len(target_layer_types) - target_num_kv_shared
target_num_layers = getattr(
target_text_config, "num_hidden_layers", len(target_layer_types)
)
num_non_shared = target_num_layers - target_num_kv_shared
type_to_target_indices: dict[str, list[int]] = defaultdict(list)
for idx, lt in enumerate(target_layer_types[:num_non_shared]):
type_to_target_indices[lt].append(idx)
@@ -128,6 +152,17 @@ class Gemma4Speculator(AutoRegressiveSpeculator):
target_idx = candidates[-1]
target_layer_name = f"{target_prefix}.{target_idx}.self_attn.attn"
attn.kv_sharing_target_layer_name = target_layer_name
# KV-cache sharing aliases the cache tensor during allocation, but
# the quantization scales live on the Attention modules themselves.
# The BF16 draft model has no quantization config, so its K/V scales
# otherwise remain at the default 1.0 while it reads the target's
# calibrated FP8 cache.
target_attn = self.vllm_config.compilation_config.static_forward_context[
target_layer_name
]
_copy_target_kv_scales(attn, target_attn)
logger.info(
"Gemma4 MTP: draft layer %d (%s) -> %s",
draft_idx,