[Kimi-K3] Add FlashInfer speculative KDA backend (#54255)
Signed-off-by: Duncan Moss <dmoss@nvidia.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
co-authored by
Roger Wang
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Thien Tran
parent
edaf3e7b61
commit
766b4ae54c
@@ -30,13 +30,17 @@ from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia import kda as nvidia_kda
|
||||
from vllm.models.kimi_k3.nvidia.kda import (
|
||||
KimiK3DeltaAttention,
|
||||
_flashinfer_kda_prefill,
|
||||
_flashkda_prefill,
|
||||
is_flashinfer_fused_kda_decode_supported,
|
||||
is_flashinfer_fused_kda_spec_decode_supported,
|
||||
is_flashinfer_recurrent_kda_prefill_supported,
|
||||
is_flashkda_supported,
|
||||
is_fused_kda_decode_supported,
|
||||
resolve_kda_spec_decode_backend,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.kda_metadata import KimiK3KDAMetadata
|
||||
from vllm.models.kimi_k3.nvidia.model import KimiLinearForCausalLM
|
||||
from vllm.models.kimi_k3.nvidia.ops import recoverssm as recoverssm_ops
|
||||
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
|
||||
@@ -90,6 +94,25 @@ def test_kda_warmup_skips_missing_metadata(monkeypatch):
|
||||
assert layer._forward(empty, empty, empty, empty, empty) is None
|
||||
|
||||
|
||||
def test_resolve_kda_spec_decode_backend(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported",
|
||||
lambda *args: True,
|
||||
)
|
||||
args = (12, 128, 4, 6, torch.bfloat16, torch.bfloat16, torch.float32, -5.0)
|
||||
assert resolve_kda_spec_decode_backend("auto", *args, False) == "flashinfer"
|
||||
assert resolve_kda_spec_decode_backend("native", *args, False) == "native"
|
||||
assert resolve_kda_spec_decode_backend("flashinfer", *args, False) == "flashinfer"
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported",
|
||||
lambda *args: False,
|
||||
)
|
||||
assert resolve_kda_spec_decode_backend("auto", *args, False) == "native"
|
||||
with pytest.raises(RuntimeError, match="packed_fused_kda_decode"):
|
||||
resolve_kda_spec_decode_backend("flashinfer", *args, False)
|
||||
|
||||
|
||||
def test_kda_recoverssm_config_state_layout():
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
@@ -1182,6 +1205,224 @@ def test_fused_kda_decode_correctness(
|
||||
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("state_dtype", [torch.float32, torch.bfloat16])
|
||||
def test_flashinfer_fused_kda_spec_decode_integration(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
state_dtype: torch.dtype,
|
||||
):
|
||||
if not is_flashinfer_fused_kda_spec_decode_supported(
|
||||
num_heads=12,
|
||||
head_dim=128,
|
||||
conv_width=4,
|
||||
num_spec=6,
|
||||
input_dtype=torch.bfloat16,
|
||||
conv_state_dtype=torch.bfloat16,
|
||||
recurrent_state_dtype=state_dtype,
|
||||
lower_bound=-5.0,
|
||||
use_recoverssm=False,
|
||||
):
|
||||
pytest.skip("FlashInfer packed fused KDA decode is not supported")
|
||||
|
||||
generator = torch.Generator(device=DEVICE).manual_seed(20260820)
|
||||
H, D, W, T = 12, 128, 4, 7
|
||||
hidden_size = H * D
|
||||
num_rows = 10
|
||||
num_slots = 16
|
||||
|
||||
x = torch.randn(
|
||||
num_rows,
|
||||
3 * hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
generator=generator,
|
||||
)
|
||||
conv_weight = 0.1 * torch.randn(
|
||||
3 * hidden_size,
|
||||
W,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
generator=generator,
|
||||
)
|
||||
fused_weight = conv_weight.view(3, hidden_size, W).transpose(1, 2).contiguous()
|
||||
raw_gate = torch.randn(
|
||||
1, num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator
|
||||
)
|
||||
raw_beta = torch.randn(
|
||||
1, num_rows, H, dtype=torch.bfloat16, device=DEVICE, generator=generator
|
||||
)
|
||||
output_gate = torch.randn(
|
||||
num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator
|
||||
)
|
||||
A_log = torch.log(torch.arange(1, H + 1, dtype=torch.float32, device=DEVICE))
|
||||
dt_bias = torch.empty(hidden_size, dtype=torch.float32, device=DEVICE).uniform_(
|
||||
-7.0, -4.0, generator=generator
|
||||
)
|
||||
norm_weight = torch.randn(
|
||||
D, dtype=torch.float32, device=DEVICE, generator=generator
|
||||
)
|
||||
norm_eps = 1e-5
|
||||
state_indices = torch.tensor(
|
||||
[
|
||||
list(range(1, T + 1)),
|
||||
list(range(T + 1, 2 * T + 1)),
|
||||
[0] * T,
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
query_start_loc = torch.tensor([0, 7, 10, 10], dtype=torch.int32, device=DEVICE)
|
||||
num_accepted_tokens = torch.tensor([1, 4, 1], dtype=torch.int32, device=DEVICE)
|
||||
conv_seed = (
|
||||
0.5
|
||||
* torch.randn(
|
||||
num_slots,
|
||||
T + W - 2,
|
||||
3 * hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
generator=generator,
|
||||
)
|
||||
).transpose(1, 2)
|
||||
state_seed = (
|
||||
0.5
|
||||
* torch.randn(
|
||||
num_slots,
|
||||
H,
|
||||
D,
|
||||
D,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE,
|
||||
generator=generator,
|
||||
)
|
||||
).to(state_dtype)
|
||||
metadata = KimiK3KDAMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_spec_decodes=2,
|
||||
num_spec_decode_tokens=num_rows,
|
||||
num_actual_tokens=num_rows,
|
||||
spec_query_start_loc=query_start_loc,
|
||||
spec_state_indices_tensor=state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.kimi_k3.nvidia.kda.get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata={"test.layer": metadata}),
|
||||
)
|
||||
|
||||
def clone_strided(tensor: torch.Tensor) -> torch.Tensor:
|
||||
clone = torch.empty_strided(
|
||||
tensor.shape, tensor.stride(), dtype=tensor.dtype, device=tensor.device
|
||||
)
|
||||
clone.copy_(tensor)
|
||||
return clone
|
||||
|
||||
def composed_reference(
|
||||
conv_state: torch.Tensor,
|
||||
recurrent_state: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
mixed_qkv = causal_conv1d_update(
|
||||
x,
|
||||
conv_state,
|
||||
conv_weight,
|
||||
bias=None,
|
||||
activation="silu",
|
||||
conv_state_indices=state_indices[:, 0],
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
max_query_len=T,
|
||||
validate_data=False,
|
||||
out=torch.empty_like(x),
|
||||
)
|
||||
q, k, v = mixed_qkv.view(num_rows, 3, H, D).unbind(1)
|
||||
recurrent_out, _ = fused_recurrent_kda_nvidia(
|
||||
q=q.unsqueeze(0),
|
||||
k=k.unsqueeze(0),
|
||||
v=v.unsqueeze(0),
|
||||
raw_g=raw_gate,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
lower_bound=-5.0,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=query_start_loc,
|
||||
ssm_state_indices=state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
)
|
||||
recurrent_float = recurrent_out.float()
|
||||
return (
|
||||
recurrent_float
|
||||
* torch.rsqrt(
|
||||
recurrent_float.square().mean(dim=-1, keepdim=True) + norm_eps
|
||||
)
|
||||
* norm_weight
|
||||
* output_gate.float().sigmoid().unsqueeze(0)
|
||||
).to(torch.bfloat16)
|
||||
|
||||
def run_flashinfer(
|
||||
conv_state: torch.Tensor,
|
||||
recurrent_state: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
layer = SimpleNamespace(
|
||||
prefix="test.layer",
|
||||
kv_cache=(conv_state.transpose(-1, -2), recurrent_state),
|
||||
kda_spec_decode_backend="flashinfer",
|
||||
decode_conv1d_weight=fused_weight,
|
||||
decode_norm_weight=norm_weight,
|
||||
gate_lower_bound=-5.0,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
o_norm=SimpleNamespace(eps=norm_eps),
|
||||
)
|
||||
KimiK3DeltaAttention._forward(
|
||||
layer,
|
||||
mixed_qkv=x,
|
||||
g1=raw_gate,
|
||||
g2=output_gate,
|
||||
beta=raw_beta,
|
||||
core_attn_out=output,
|
||||
)
|
||||
return output
|
||||
|
||||
expected_conv = clone_strided(conv_seed)
|
||||
expected_state = state_seed.clone()
|
||||
expected = composed_reference(expected_conv, expected_state)
|
||||
actual_conv = clone_strided(conv_seed)
|
||||
actual_state = state_seed.clone()
|
||||
output = torch.empty_like(expected)
|
||||
actual = run_flashinfer(actual_conv, actual_state, output)
|
||||
|
||||
assert actual.data_ptr() == output.data_ptr()
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=3e-2)
|
||||
assert torch.count_nonzero(actual[:, :7]) > 0
|
||||
torch.testing.assert_close(actual_state[0], state_seed[0], atol=0, rtol=0)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = run_flashinfer(actual_conv, actual_state, output)
|
||||
|
||||
actual_conv.copy_(conv_seed)
|
||||
actual_state.copy_(state_seed)
|
||||
query_start_loc.copy_(torch.tensor([0, 6, 10, 10], device=DEVICE))
|
||||
num_accepted_tokens.copy_(torch.tensor([3, 2, 1], device=DEVICE))
|
||||
expected_conv.copy_(conv_seed)
|
||||
expected_state.copy_(state_seed)
|
||||
expected = composed_reference(expected_conv, expected_state)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert captured.data_ptr() == output.data_ptr()
|
||||
torch.testing.assert_close(output, expected, atol=2e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
|
||||
|
||||
|
||||
def test_fused_kda_decode_rejects_speculative_conv_state():
|
||||
assert not is_fused_kda_decode_supported(
|
||||
num_heads=12,
|
||||
|
||||
@@ -55,8 +55,10 @@ from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
from vllm.utils.flashinfer import (
|
||||
flashinfer_fused_kda_decode,
|
||||
flashinfer_packed_fused_kda_decode,
|
||||
flashinfer_recurrent_kda,
|
||||
has_flashinfer_fused_kda_decode,
|
||||
has_flashinfer_packed_fused_kda_decode,
|
||||
has_flashinfer_recurrent_kda,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
@@ -214,6 +216,73 @@ def is_flashinfer_fused_kda_decode_supported(
|
||||
)
|
||||
|
||||
|
||||
def is_flashinfer_fused_kda_spec_decode_supported(
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
conv_width: int,
|
||||
num_spec: int,
|
||||
input_dtype: torch.dtype,
|
||||
conv_state_dtype: torch.dtype,
|
||||
recurrent_state_dtype: torch.dtype,
|
||||
lower_bound: float | None,
|
||||
use_recoverssm: bool,
|
||||
) -> bool:
|
||||
return (
|
||||
has_flashinfer_packed_fused_kda_decode()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
and num_heads in (12, 24, 32, 48, 96)
|
||||
and head_dim == 128
|
||||
and conv_width == 4
|
||||
and 1 <= num_spec <= 7
|
||||
and input_dtype == torch.bfloat16
|
||||
and conv_state_dtype == torch.bfloat16
|
||||
and recurrent_state_dtype in (torch.float32, torch.bfloat16)
|
||||
and lower_bound is not None
|
||||
and lower_bound < 0
|
||||
and not use_recoverssm
|
||||
and not is_conv_state_dim_first()
|
||||
)
|
||||
|
||||
|
||||
def resolve_kda_spec_decode_backend(
|
||||
backend: str,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
conv_width: int,
|
||||
num_spec: int,
|
||||
input_dtype: torch.dtype,
|
||||
conv_state_dtype: torch.dtype,
|
||||
recurrent_state_dtype: torch.dtype,
|
||||
lower_bound: float | None,
|
||||
use_recoverssm: bool,
|
||||
) -> str:
|
||||
if backend not in ("auto", "native", "flashinfer"):
|
||||
raise ValueError(f"Unsupported KDA speculative decode backend: {backend}")
|
||||
supported = is_flashinfer_fused_kda_spec_decode_supported(
|
||||
num_heads,
|
||||
head_dim,
|
||||
conv_width,
|
||||
num_spec,
|
||||
input_dtype,
|
||||
conv_state_dtype,
|
||||
recurrent_state_dtype,
|
||||
lower_bound,
|
||||
use_recoverssm,
|
||||
)
|
||||
if backend == "flashinfer" and not supported:
|
||||
raise RuntimeError(
|
||||
"FlashInfer packed fused KDA decode requires its "
|
||||
"packed_fused_kda_decode API, CUDA SM10x, bfloat16 inputs and "
|
||||
"convolution state, float32 or bfloat16 recurrent state, D=128, "
|
||||
"W=4, a bounded gate, 1-7 speculative tokens, a supported head "
|
||||
"count, the SD convolution-state layout, and RecoverSSM disabled."
|
||||
)
|
||||
if supported and backend != "native":
|
||||
logger.info_once("Using FlashInfer packed fused KDA speculative decode.")
|
||||
return "flashinfer"
|
||||
return "native"
|
||||
|
||||
|
||||
def resolve_kda_decode_backend(
|
||||
backend: str,
|
||||
num_heads: int,
|
||||
@@ -537,6 +606,13 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
self.projection_size = self.head_dim * self.num_heads
|
||||
self.local_projection_size = divide(self.projection_size, self.tp_size)
|
||||
self.conv_size = kda_config["short_conv_kernel_size"]
|
||||
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
|
||||
if self.gate_lower_bound is not None:
|
||||
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
|
||||
"KDA gate lower bound must be in "
|
||||
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
|
||||
f"Got {self.gate_lower_bound}."
|
||||
)
|
||||
assert kda_config.get("use_full_rank_gate", False), (
|
||||
"KimiK3DeltaAttention requires a full-rank gate"
|
||||
)
|
||||
@@ -618,8 +694,28 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
conv_state_dtype,
|
||||
recurrent_state_dtype,
|
||||
)
|
||||
spec_decode_backend = (
|
||||
additional_config.get("kda_spec_decode_backend", "auto")
|
||||
if isinstance(additional_config, dict)
|
||||
else "auto"
|
||||
)
|
||||
self.kda_spec_decode_backend = resolve_kda_spec_decode_backend(
|
||||
spec_decode_backend,
|
||||
self.local_num_heads,
|
||||
self.head_dim,
|
||||
self.conv_size,
|
||||
self.num_spec,
|
||||
vllm_config.model_config.dtype,
|
||||
conv_state_dtype,
|
||||
recurrent_state_dtype,
|
||||
self.gate_lower_bound,
|
||||
self.use_recoverssm,
|
||||
)
|
||||
decode_conv1d_weight = None
|
||||
if self.kda_decode_backend != "triton":
|
||||
if (
|
||||
self.kda_decode_backend != "triton"
|
||||
or self.kda_spec_decode_backend == "flashinfer"
|
||||
):
|
||||
decode_conv1d_weight = torch.empty(
|
||||
3,
|
||||
self.conv_size,
|
||||
@@ -648,14 +744,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
)
|
||||
set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)})
|
||||
|
||||
self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None)
|
||||
if self.gate_lower_bound is not None:
|
||||
assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, (
|
||||
"KDA gate lower bound must be in "
|
||||
f"[{_KDA_GATE_LOGBOUND_MIN}, 0). "
|
||||
f"Got {self.gate_lower_bound}."
|
||||
)
|
||||
|
||||
backend = (
|
||||
additional_config.get("kda_prefill_backend", "auto")
|
||||
if isinstance(additional_config, dict)
|
||||
@@ -860,6 +948,39 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
if not is_conv_state_dim_first():
|
||||
conv_state = conv_state.transpose(-1, -2)
|
||||
|
||||
if (
|
||||
self.kda_spec_decode_backend == "flashinfer"
|
||||
and has_spec_decode
|
||||
and m.num_prefills == 0
|
||||
and m.num_decodes == 0
|
||||
):
|
||||
assert self.decode_conv1d_weight is not None
|
||||
assert self.decode_norm_weight is not None
|
||||
assert spec_state_indices_tensor is not None
|
||||
assert spec_query_start_loc is not None
|
||||
assert num_accepted_tokens is not None
|
||||
assert self.gate_lower_bound is not None
|
||||
# "Packed" is FlashInfer's name for the varlen T>1 speculative path.
|
||||
flashinfer_packed_fused_kda_decode(
|
||||
x=mixed_qkv,
|
||||
weight=self.decode_conv1d_weight,
|
||||
conv_state=conv_state,
|
||||
raw_gate=g1,
|
||||
raw_beta=beta,
|
||||
A_log=self.A_log,
|
||||
dt_bias=self.dt_bias,
|
||||
state_indices=spec_state_indices_tensor,
|
||||
query_start_loc=spec_query_start_loc,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
state=recurrent_state,
|
||||
output_gate=g2[:num_actual_tokens],
|
||||
norm_weight=self.decode_norm_weight,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
norm_eps=self.o_norm.eps,
|
||||
output=core_attn_out[:, :num_actual_tokens],
|
||||
)
|
||||
return
|
||||
|
||||
if (
|
||||
self.kda_decode_backend != "triton"
|
||||
and self.decode_conv1d_weight is not None
|
||||
|
||||
@@ -313,6 +313,9 @@ flashinfer_xqa_batch_decode_with_kv_cache = _lazy_import_wrapper(
|
||||
"flashinfer.decode",
|
||||
"xqa_batch_decode_with_kv_cache",
|
||||
)
|
||||
flashinfer_packed_fused_kda_decode = _lazy_import_wrapper(
|
||||
"flashinfer", "packed_fused_kda_decode"
|
||||
)
|
||||
flashinfer_recurrent_kda = _lazy_import_wrapper(
|
||||
"flashinfer.kda",
|
||||
"recurrent_kda",
|
||||
@@ -337,6 +340,15 @@ def has_flashinfer_comm() -> bool:
|
||||
return has_flashinfer() and importlib.util.find_spec("flashinfer.comm") is not None
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_packed_fused_kda_decode() -> bool:
|
||||
"""Return whether FlashInfer's packed fused KDA decode API is available."""
|
||||
if not has_flashinfer():
|
||||
return False
|
||||
module = _get_submodule("flashinfer")
|
||||
return bool(module and callable(getattr(module, "packed_fused_kda_decode", None)))
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_nvlink_two_sided() -> bool:
|
||||
"""Return `True` if FlashInfer mnnvl all2all is available."""
|
||||
|
||||
Reference in New Issue
Block a user