[Kernel] Resubmit PR 48666 - Gemma4 FP8 KV FA4 head dim 512 backend selection (#53175)
Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com>
This commit is contained in:
@@ -24,6 +24,27 @@ from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _copy_target_kv_scales(attn: nn.Module, target_attn: nn.Module) -> None:
|
||||
"""Copy target KV scales while preserving their tensor representation.
|
||||
|
||||
Default attention scales are scalar buffers, while some quantization
|
||||
methods replace them with length-one or per-head parameters. Preserve the
|
||||
draft layer's registration type so the shared KV cache is interpreted with
|
||||
the target's values and shapes without aliasing target parameters.
|
||||
"""
|
||||
for scale_name in ("_k_scale", "_v_scale"):
|
||||
draft_scale = getattr(attn, scale_name)
|
||||
target_scale = getattr(target_attn, scale_name)
|
||||
scale = target_scale.detach().clone()
|
||||
if isinstance(draft_scale, nn.Parameter):
|
||||
scale = nn.Parameter(scale, requires_grad=False)
|
||||
setattr(attn, scale_name, scale)
|
||||
for scale_name in ("_k_scale_float", "_v_scale_float"):
|
||||
setattr(attn, scale_name, getattr(target_attn, scale_name))
|
||||
for scale_name in ("_k_scale_cpu", "_v_scale_cpu"):
|
||||
getattr(attn, scale_name).copy_(getattr(target_attn, scale_name))
|
||||
|
||||
|
||||
class Gemma4Speculator(AutoRegressiveSpeculator):
|
||||
@property
|
||||
def advance_draft_positions(self) -> bool:
|
||||
@@ -77,7 +98,7 @@ class Gemma4Speculator(AutoRegressiveSpeculator):
|
||||
model: nn.Module,
|
||||
target_attn_layer_names: set[str],
|
||||
) -> None:
|
||||
"""Wire draft layers to share KV with the target model.
|
||||
"""Wire draft layers to share KV and KV scales with the target model.
|
||||
|
||||
Each draft decoder layer is mapped to the last non-KV-shared
|
||||
target layer of the same attention type (sliding or full).
|
||||
@@ -92,7 +113,10 @@ class Gemma4Speculator(AutoRegressiveSpeculator):
|
||||
return
|
||||
|
||||
target_num_kv_shared = getattr(target_text_config, "num_kv_shared_layers", 0)
|
||||
num_non_shared = len(target_layer_types) - target_num_kv_shared
|
||||
target_num_layers = getattr(
|
||||
target_text_config, "num_hidden_layers", len(target_layer_types)
|
||||
)
|
||||
num_non_shared = target_num_layers - target_num_kv_shared
|
||||
type_to_target_indices: dict[str, list[int]] = defaultdict(list)
|
||||
for idx, lt in enumerate(target_layer_types[:num_non_shared]):
|
||||
type_to_target_indices[lt].append(idx)
|
||||
@@ -128,6 +152,17 @@ class Gemma4Speculator(AutoRegressiveSpeculator):
|
||||
target_idx = candidates[-1]
|
||||
target_layer_name = f"{target_prefix}.{target_idx}.self_attn.attn"
|
||||
attn.kv_sharing_target_layer_name = target_layer_name
|
||||
|
||||
# KV-cache sharing aliases the cache tensor during allocation, but
|
||||
# the quantization scales live on the Attention modules themselves.
|
||||
# The BF16 draft model has no quantization config, so its K/V scales
|
||||
# otherwise remain at the default 1.0 while it reads the target's
|
||||
# calibrated FP8 cache.
|
||||
target_attn = self.vllm_config.compilation_config.static_forward_context[
|
||||
target_layer_name
|
||||
]
|
||||
_copy_target_kv_scales(attn, target_attn)
|
||||
|
||||
logger.info(
|
||||
"Gemma4 MTP: draft layer %d (%s) -> %s",
|
||||
draft_idx,
|
||||
|
||||
Reference in New Issue
Block a user