[Kernel] Add router GEMM kernels for (5120, 256) and BF16 output to ll_bf16_gemm (#62042)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Lain <fusiyuan2000@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
co-authored by
Lain
Codex
Claude Opus 5.5
vllm-agent
parent
153a17166e
commit
596e530675
@@ -566,7 +566,7 @@ def test_invalid_output_dtype():
|
||||
with pytest.raises(ValueError, match="output_dtype=torch.float32"):
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
|
||||
|
||||
ll_bf16_gemm(a, b, output_dtype=torch.bfloat16)
|
||||
ll_bf16_gemm(a, b, output_dtype=torch.float16)
|
||||
|
||||
|
||||
def test_cache_miss_compiles_dotprod():
|
||||
@@ -581,5 +581,29 @@ def test_cache_miss_compiles_dotprod():
|
||||
_assert_close(out, _ref(a, b), context="cache miss dotprod")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [1, 2, 8, 16])
|
||||
@pytest.mark.parametrize("N", [1792, 3584])
|
||||
@torch.inference_mode()
|
||||
def test_bf16_output_and_fp32_cache_coexist(M, N):
|
||||
"""BF16 epilogues retain FP32 accumulation and can replay with new inputs."""
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
|
||||
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(M, 5120, device="cuda", dtype=torch.bfloat16)
|
||||
w = torch.randn(N, 5120, device="cuda", dtype=torch.bfloat16) * 0.01
|
||||
for dtype in (torch.float32, torch.bfloat16):
|
||||
actual = ll_bf16_gemm(x, w, output_dtype=dtype)
|
||||
assert actual.dtype == dtype
|
||||
reference = (x.double() @ w.double().T).to(dtype)
|
||||
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = ll_bf16_gemm(x, w, output_dtype=torch.bfloat16)
|
||||
x.normal_()
|
||||
graph.replay()
|
||||
reference = (x.double() @ w.double().T).bfloat16()
|
||||
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
Reference in New Issue
Block a user