[K3 Perf] Optimize k3 dspark fused kv, 4.5~4.6x kernel performance improvement (#50585)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
0df620d429
commit
56a4b63d44
@@ -107,6 +107,7 @@ def test_dspark_markov_head_is_replicated(
|
||||
@pytest.mark.cpu_test
|
||||
def test_k3_dspark_uses_replicated_markov_head(monkeypatch: pytest.MonkeyPatch):
|
||||
markov_head_calls = []
|
||||
context_kv_proj_calls = []
|
||||
|
||||
class DummyModule(nn.Module):
|
||||
def __init__(self, *args, **kwargs):
|
||||
@@ -116,8 +117,13 @@ def test_k3_dspark_uses_replicated_markov_head(monkeypatch: pytest.MonkeyPatch):
|
||||
markov_head_calls.append((args, kwargs))
|
||||
return DummyModule()
|
||||
|
||||
def make_context_kv_proj(*args, **kwargs):
|
||||
context_kv_proj_calls.append((args, kwargs))
|
||||
return DummyModule()
|
||||
|
||||
monkeypatch.setattr(dspark_mla, "get_draft_quant_config", lambda _: None)
|
||||
monkeypatch.setattr(dspark_mla, "ReplicatedLinear", DummyModule)
|
||||
monkeypatch.setattr(dspark_mla, "MergedColumnParallelLinear", make_context_kv_proj)
|
||||
monkeypatch.setattr(dspark_mla, "RMSNorm", DummyModule)
|
||||
monkeypatch.setattr(dspark_mla, "K3DSparkDecoderLayer", DummyModule)
|
||||
monkeypatch.setattr(dspark_mla, "DSparkMarkovHead", make_markov_head)
|
||||
@@ -126,6 +132,8 @@ def test_k3_dspark_uses_replicated_markov_head(monkeypatch: pytest.MonkeyPatch):
|
||||
target_hidden_size=16,
|
||||
num_target_layers=2,
|
||||
hidden_size=8,
|
||||
kv_lora_rank=3,
|
||||
qk_rope_head_dim=1,
|
||||
rms_norm_eps=1e-6,
|
||||
num_hidden_layers=1,
|
||||
vocab_size=128,
|
||||
@@ -135,9 +143,48 @@ def test_k3_dspark_uses_replicated_markov_head(monkeypatch: pytest.MonkeyPatch):
|
||||
vllm_config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(hf_config=config)
|
||||
)
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=16),
|
||||
)
|
||||
|
||||
K3DSparkModel(vllm_config=vllm_config, start_layer_id=0, prefix="model")
|
||||
|
||||
assert len(markov_head_calls) == 1
|
||||
assert context_kv_proj_calls == [
|
||||
(
|
||||
(8, [4]),
|
||||
{
|
||||
"bias": False,
|
||||
"return_bias": False,
|
||||
"quant_config": None,
|
||||
"prefix": "model.layers.0.self_attn.fused_qkv_a_proj",
|
||||
"disable_tp": True,
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_context_kv_weights_are_loaded_as_merged_linear_shards():
|
||||
weights = [
|
||||
(
|
||||
"layers.0.self_attn.kv_a_proj_with_mqa.weight_packed",
|
||||
torch.arange(4),
|
||||
),
|
||||
(
|
||||
"layers.1.self_attn.kv_a_proj_with_mqa.weight_scale",
|
||||
torch.tensor(0.5),
|
||||
),
|
||||
]
|
||||
|
||||
duplicated = dspark_mla._duplicate_context_kv_weights(weights, 2)
|
||||
mapped = list(K3DSparkForCausalLM.hf_to_vllm_mapper.apply(duplicated))
|
||||
|
||||
assert [name for name, _ in mapped] == [
|
||||
"model.layers.0.self_attn.fused_qkv_a_proj.weight_packed",
|
||||
"model.context_kv_proj.weight_packed",
|
||||
"model.layers.1.self_attn.fused_qkv_a_proj.weight_scale",
|
||||
"model.context_kv_proj.weight_scale",
|
||||
]
|
||||
assert [weight.shard_id for _, weight in mapped] == [1, 0, 1, 1]
|
||||
assert mapped[0][1].data_ptr() == mapped[1][1].data_ptr()
|
||||
assert mapped[2][1].data_ptr() == mapped[3][1].data_ptr()
|
||||
|
||||
Reference in New Issue
Block a user