[Bugfix] Fix support for dimension like integers and ScalarType (#9299)
This commit is contained in:
@@ -69,11 +69,11 @@ def check_full_graph_support(model,
|
||||
os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(optimization_level)
|
||||
os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
|
||||
|
||||
# Inductor doesn't support fp8/gptq_marlin_24 yet.
|
||||
# Inductor doesn't support fp8 and the base meta llama uses too
|
||||
# much memory.
|
||||
quantization = model_kwargs.get("quantization")
|
||||
if (quantization == "fp8" or quantization == "gptq_marlin"
|
||||
or quantization == "gptq_marlin_24"
|
||||
) and optimization_level >= CompilationLevel.INDUCTOR:
|
||||
if ((quantization == "fp8" or model == "meta-llama/Meta-Llama-3-8B")
|
||||
and optimization_level >= CompilationLevel.INDUCTOR):
|
||||
return
|
||||
|
||||
prompts = [
|
||||
|
||||
@@ -80,7 +80,7 @@ def machete_quantize_and_pack(w: torch.Tensor,
|
||||
w_q = w_q.t().contiguous().t() # convert to col major
|
||||
w_q_machete = ops.machete_prepack_B(w_q, wtype)
|
||||
|
||||
opcheck(torch.ops._C.machete_prepack_B, (w_q, wtype))
|
||||
opcheck(torch.ops._C.machete_prepack_B, (w_q, wtype.id))
|
||||
|
||||
return w_ref, w_q_machete, w_s, w_zp
|
||||
|
||||
@@ -153,9 +153,10 @@ def test_machete_all_schedules(shape, atype: torch.dtype,
|
||||
schedule=schedule,
|
||||
)
|
||||
|
||||
opcheck(torch.ops._C.machete_gemm,
|
||||
(a, w_q_machete, wtype, w_s, maybe_convert_zeropoints(
|
||||
w_zp, w_s), group_size, None, None, None, schedule))
|
||||
opcheck(
|
||||
torch.ops._C.machete_gemm,
|
||||
(a, w_q_machete, wtype.id, w_s, maybe_convert_zeropoints(
|
||||
w_zp, w_s), group_size, None, None, None, schedule))
|
||||
|
||||
# Relax atol as our reduction dim becomes larger (more rounding error)
|
||||
# Relax atol when we have zeropoints since the way machete applies
|
||||
|
||||
@@ -225,7 +225,7 @@ def test_gptq_marlin_gemm(
|
||||
opcheck(
|
||||
torch.ops._C.gptq_marlin_gemm,
|
||||
(a_input, marlin_q_w, marlin_s, marlin_zp, g_idx, sort_indices,
|
||||
workspace.scratch, quant_type, a_input.shape[0], b_weight.shape[1],
|
||||
workspace.scratch, quant_type.id, a_input.shape[0], b_weight.shape[1],
|
||||
a_input.shape[1], is_k_full, False, use_fp32_reduce),
|
||||
test_utils=DEFAULT_OPCHECK_TEST_UTILS)
|
||||
|
||||
@@ -254,6 +254,16 @@ def test_gptq_marlin_gemm(
|
||||
assert max_diff < 0.04
|
||||
|
||||
|
||||
# TODO: find better way to test this?
|
||||
@torch.compile(fullgraph=True)
|
||||
def marlin_24_gemm_tester(a_input, marlin_24_q_w_comp, marlin_24_meta,
|
||||
marlin_24_s, scratch, quant_type, size_m, size_n,
|
||||
size_k):
|
||||
return ops.gptq_marlin_24_gemm(a_input, marlin_24_q_w_comp, marlin_24_meta,
|
||||
marlin_24_s, scratch, quant_type, size_m,
|
||||
size_n, size_k)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
|
||||
reason="Marlin is not supported on this GPU type.")
|
||||
@pytest.mark.parametrize("k_chunk", MARLIN_24_K_CHUNKS)
|
||||
@@ -282,11 +292,11 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, quant_type, group_size,
|
||||
|
||||
opcheck(torch.ops._C.gptq_marlin_24_gemm,
|
||||
(a_input, marlin_24_q_w_comp, marlin_24_meta, marlin_24_s,
|
||||
workspace_24.scratch, quant_type, a_input.shape[0],
|
||||
workspace_24.scratch, quant_type.id, a_input.shape[0],
|
||||
b_weight.shape[1], a_input.shape[1]),
|
||||
test_utils=DEFAULT_OPCHECK_TEST_UTILS)
|
||||
|
||||
output = ops.gptq_marlin_24_gemm(
|
||||
output = marlin_24_gemm_tester(
|
||||
a_input,
|
||||
marlin_24_q_w_comp,
|
||||
marlin_24_meta,
|
||||
|
||||
@@ -240,8 +240,8 @@ def test_fused_marlin_moe(
|
||||
requires_grad=False)
|
||||
opcheck(torch.ops._moe_C.marlin_gemm_moe,
|
||||
(a, qweight1, sorted_token_ids, topk_weights, topk_ids,
|
||||
scales1, zp, g_idx1, sort_indices1, workspace, quant_type, m,
|
||||
2 * n, k, True, e, topk, block_size_m, True, False))
|
||||
scales1, zp, g_idx1, sort_indices1, workspace, quant_type.id,
|
||||
m, 2 * n, k, True, e, topk, block_size_m, True, False))
|
||||
|
||||
|
||||
@pytest.mark.skip("This test is here for the sake of debugging, "
|
||||
|
||||
@@ -32,5 +32,5 @@ def test_scalar_type_min_max(type_tuple):
|
||||
max = torch.iinfo(torch_type).max
|
||||
|
||||
print(t, min, max, t.min(), t.max())
|
||||
assert min == t.min()
|
||||
assert max == t.max()
|
||||
assert min == t.min(), f"min: {min} != {t.min()}"
|
||||
assert max == t.max(), f"max: {max} != {t.max()}"
|
||||
|
||||
Reference in New Issue
Block a user