[
    {
        "load_fn": "exllamav2",
        "fwd_fn": "exllamav2",
        "label": "EXL2 2.4bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl2/2.4bpw/"
    },
    {
        "load_fn": "exllamav2",
        "fwd_fn": "exllamav2",
        "label": "EXL2 2.8bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl2/2.8bpw/"
    },
    {
        "load_fn": "exllamav2",
        "fwd_fn": "exllamav2",
        "label": "EXL2 3.5bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl2/3.5bpw/"
    },
    {
        "load_fn": "exllamav2",
        "fwd_fn": "exllamav2",
        "label": "EXL2 4.0bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl2/4.0bpw/"
    },
    {
        "load_fn": "exllamav2",
        "fwd_fn": "exllamav2",
        "label": "EXL2 4.5bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl2/4.5bpw/"
    },
    {
        "load_fn": "exllamav2",
        "fwd_fn": "exllamav2",
        "label": "EXL2 5.0bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl2/5.0bpw/"
    },
    {
        "load_fn": "exllamav2",
        "fwd_fn": "exllamav2",
        "label": "EXL2 6.0bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl2/6.0bpw/"
    }
]