[Kernel] Add router GEMM kernels for (5120, 256) and BF16 output to ll_bf16_gemm (#62042)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Lain <fusiyuan2000@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-10-16 15:39:46 -07:00
committed by GitHub
co-authored by Lain Codex Claude Opus 5.5 vllm-agent
parent 153a17166e
commit 596e530675
12 changed files with 258 additions and 64 deletions
+25 -1
View File
@@ -566,7 +566,7 @@ def test_invalid_output_dtype():
with pytest.raises(ValueError, match="output_dtype=torch.float32"):
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
ll_bf16_gemm(a, b, output_dtype=torch.bfloat16)
ll_bf16_gemm(a, b, output_dtype=torch.float16)
def test_cache_miss_compiles_dotprod():
@@ -581,5 +581,29 @@ def test_cache_miss_compiles_dotprod():
_assert_close(out, _ref(a, b), context="cache miss dotprod")
@pytest.mark.parametrize("M", [1, 2, 8, 16])
@pytest.mark.parametrize("N", [1792, 3584])
@torch.inference_mode()
def test_bf16_output_and_fp32_cache_coexist(M, N):
"""BF16 epilogues retain FP32 accumulation and can replay with new inputs."""
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
torch.manual_seed(42)
x = torch.randn(M, 5120, device="cuda", dtype=torch.bfloat16)
w = torch.randn(N, 5120, device="cuda", dtype=torch.bfloat16) * 0.01
for dtype in (torch.float32, torch.bfloat16):
actual = ll_bf16_gemm(x, w, output_dtype=dtype)
assert actual.dtype == dtype
reference = (x.double() @ w.double().T).to(dtype)
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = ll_bf16_gemm(x, w, output_dtype=torch.bfloat16)
x.normal_()
graph.replay()
reference = (x.double() @ w.double().T).bfloat16()
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
if __name__ == "__main__":
pytest.main([__file__, "-v"])