[Kernel][Perf] Register-resident path for per-token-group 8-bit quant (#55330)
Signed-off-by: chao.huan <chao.huan@nio.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
@@ -117,7 +117,18 @@ if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
warmup_iters, bench_iters = args.warmup_iters, args.bench_iters
|
||||
|
||||
shapes = [(32, 128), (64, 256), (16, 512)]
|
||||
shapes = [
|
||||
(32, 128),
|
||||
(64, 256),
|
||||
(16, 512),
|
||||
# Serving-sized activations, where this op is memory-bound and its
|
||||
# schedule dominates: a prefill chunk and a decode batch at two
|
||||
# common hidden sizes.
|
||||
(1024, 6144),
|
||||
(8192, 6144),
|
||||
(1024, 7168),
|
||||
(8192, 7168),
|
||||
]
|
||||
group_sizes = [64, 128]
|
||||
|
||||
dtypes = ["fp8", "int8"] if args.dtype == "both" else [args.dtype]
|
||||
|
||||
Reference in New Issue
Block a user