turboderp-org/exllamav3
1
0
Fork 0
mirror of https://github.com/turboderp-org/exllamav3.git synced 2026-03-15 00:07:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

HumanEval: Add max batch size arg

Browse Source
This commit is contained in:
turboderp
2025-12-05 13:21:07 +01:00
parent bb43823e32
commit 0a629cf70a
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
eval/humaneval.py
View File

@@ -114,7 +114,7 @@ def main(args):
generator = Generator(
model = model,
cache = cache,
max_batch_size = 256,
max_batch_size = args.max_batch_size,
tokenizer = tokenizer
)
sampler = ComboSampler(
@@ -214,6 +214,7 @@ if __name__ == "__main__":
parser.add_argument("-topk", "--top_k", type = int, help = "Top-k sampling, default: 0 (disabled)", default = 0)
parser.add_argument("-topp", "--top_p", type = float, help = "Top-p sampling, default: 0.6", default = 0.6)
parser.add_argument("-templast", "--temp_last", action = "store_true", help = "Use temperature last")
parser.add_argument("--max_tokens", type = int, default = 768, help = "Max number of tokens for each completion")
parser.add_argument("--max_tokens", type = int, default = 1024, help = "Max number of tokens for each completion")
parser.add_argument("-mbs", "--max_batch_size", type = int, default = 256, help = "Max batch size")
_args = parser.parse_args()
main(_args)