Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -56,16 +56,24 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
|
||||
rope_position_offset=0,
|
||||
),
|
||||
compressed_key_cache=SimpleNamespace(kv_cache=torch.empty(0)),
|
||||
use_fused_pre_indexer=True,
|
||||
rotary_emb=SimpleNamespace(cos_sin_cache=torch.empty(0)),
|
||||
q_layernorm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
|
||||
k_layernorm=SimpleNamespace(weight=torch.ones(1)),
|
||||
compress_ratio=2,
|
||||
)
|
||||
attn = SimpleNamespace(
|
||||
use_fused_qsa_prepare=True,
|
||||
kv_cache=torch.empty(0, 1, 1, 2),
|
||||
kv_cache_dtype="auto",
|
||||
q_norm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
|
||||
k_norm=SimpleNamespace(weight=torch.ones(1)),
|
||||
_k_scale_float=1.0,
|
||||
_v_scale_float=1.0,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
indexer_qsa,
|
||||
"qsa_pre_indexer",
|
||||
"qsa_prepare",
|
||||
lambda *args, **kwargs: updates.append((args, kwargs)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
@@ -79,11 +87,14 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
|
||||
lambda *args, **kwargs: selections.append((args, kwargs)),
|
||||
)
|
||||
|
||||
actual = indexer_qsa.QSAIndexer.forward(
|
||||
actual, _ = indexer_qsa.QSAIndexer.forward(
|
||||
indexer,
|
||||
torch.zeros(2, 2),
|
||||
torch.tensor([7, 8]),
|
||||
rows,
|
||||
attn=attn,
|
||||
qkv=torch.zeros(2, 4),
|
||||
slot_mapping=torch.arange(2),
|
||||
)
|
||||
|
||||
assert actual is rows
|
||||
@@ -573,7 +584,6 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
|
||||
_metadata=lambda: (raw_metadata, compressed_metadata),
|
||||
skip_topk=True,
|
||||
index_kv_heads=1,
|
||||
use_fused_pre_indexer=False,
|
||||
index_n_heads=1,
|
||||
index_head_dim=64,
|
||||
indexer_dtype=torch.bfloat16,
|
||||
@@ -596,6 +606,7 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
|
||||
torch.cat((torch.ones_like(padded_keys), padded_keys), dim=-1),
|
||||
torch.zeros(8, dtype=torch.long, device=device),
|
||||
torch.full((5, 5), -1, dtype=torch.int32, device=device),
|
||||
attn=SimpleNamespace(use_fused_qsa_prepare=False),
|
||||
)
|
||||
torch.testing.assert_close(raw_cache[0, :, 0], keys[[4, 1, 2, 3]])
|
||||
expected_compressed = torch.zeros_like(compressed_cache)
|
||||
|
||||
Reference in New Issue
Block a user