[
    {
        "load_fn": "exllamav3",
        "fwd_fn": "exllamav3",
        "label": "EXL3 1.6bpw H3",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl3_x/1.6bpw_H3/"
    },
    {
        "load_fn": "exllamav3",
        "fwd_fn": "exllamav3",
        "label": "EXL3 1.6bpw H3",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl3/1.6bpw_H3/"
    },
    {
        "load_fn": "exllamav3",
        "fwd_fn": "exllamav3",
        "label": "EXL3 1.8bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl3/1.8bpw/"
    },
    {
        "load_fn": "exllamav3",
        "fwd_fn": "exllamav3",
        "label": "EXL3 2.0bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl3/2.0bpw/"
    },
    {
        "load_fn": "exllamav3",
        "fwd_fn": "exllamav3",
        "label": "EXL3 2.25bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl3/2.25bpw/"
    },
    {
        "load_fn": "exllamav3",
        "fwd_fn": "exllamav3",
        "label": "EXL3 2.5bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl3/2.5bpw/"
    },
    {
        "load_fn": "exllamav3",
        "fwd_fn": "exllamav3",
        "label": "EXL3 3.0bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl3/3.0bpw/"
    },
    {
        "load_fn": "exllamav3",
        "fwd_fn": "exllamav3",
        "label": "EXL3 3.5bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl3/3.5bpw/"
    },
    {
        "load_fn": "exllamav3",
        "fwd_fn": "exllamav3",
        "label": "EXL3 4.0bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl3/4.0bpw/"
    },
    {
        "load_fn": "exllamav3",
        "fwd_fn": "exllamav3",
        "label": "EXL3 4.5bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl3/4.5bpw/"
    },
    {
        "load_fn": "exllamav3",
        "fwd_fn": "exllamav3",
        "label": "EXL3 5.0bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl3/5.0bpw/"
    },
    {
        "load_fn": "exllamav3",
        "fwd_fn": "exllamav3",
        "label": "EXL3 6.0bpw H6",
        "model_dir": "/mnt/str/eval_models/llama3.1-70b-instruct/exl3/6.0bpw/"
    }
]


