From 54d9cb79ec495a2122f7335a9747f201557ae63f Mon Sep 17 00:00:00 2001 From: Iwan Kawrakow Date: Sat, 22 Mar 2025 14:32:23 +0200 Subject: [PATCH] Adding ability to use THP on Linux --- common/common.cpp | 6 +++++ common/common.h | 1 + include/llama.h | 1 + src/llama.cpp | 66 ++++++++++++++++++++++++++++++++++++++++------- 4 files changed, 65 insertions(+), 9 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index e62944b9..5b3d60b3 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -993,6 +993,10 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa params.use_mmap = false; return true; } + if (arg == "-thp" || arg == "--transparent-huge-pages") { + params.use_thp = true; + return true; + } if (arg == "--numa") { CHECK_ARG std::string value(argv[i]); @@ -2316,6 +2320,7 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params & mparams.use_mlock = params.use_mlock; mparams.check_tensors = params.check_tensors; mparams.repack_tensors = params.repack_tensors; + mparams.use_thp = params.use_thp; if (params.kv_overrides.empty()) { mparams.kv_overrides = NULL; } else { @@ -3371,6 +3376,7 @@ void yaml_dump_non_result_info(FILE * stream, const gpt_params & params, const l fprintf(stream, "n_probs: %d # only used by server binary, default: 0\n", sparams.n_probs); fprintf(stream, "no_mmap: %s # default: false\n", !params.use_mmap ? "true" : "false"); fprintf(stream, "repack: %s # default: false\n", params.repack_tensors ? "true" : "false"); + fprintf(stream, "use_thp: %s # default: false\n", params.use_thp ? "true" : "false"); fprintf(stream, "penalize_nl: %s # default: false\n", sparams.penalize_nl ? "true" : "false"); fprintf(stream, "ppl_output_type: %d # default: 0\n", params.ppl_output_type); fprintf(stream, "ppl_stride: %d # default: 0\n", params.ppl_stride); diff --git a/common/common.h b/common/common.h index f6a55885..79c5fe96 100644 --- a/common/common.h +++ b/common/common.h @@ -194,6 +194,7 @@ struct gpt_params { bool warmup = true; // warmup run bool check_tensors = false; // validate tensor data bool repack_tensors = false; // repack tensors if interleaved variant is available + bool use_thp = false; // use transparent huge pages (linux only) std::string cache_type_k = "f16"; // KV cache data type for the K std::string cache_type_v = "f16"; // KV cache data type for the V diff --git a/include/llama.h b/include/llama.h index 1b9c47e9..38822dbe 100644 --- a/include/llama.h +++ b/include/llama.h @@ -345,6 +345,7 @@ extern "C" { bool use_mlock; // force system to keep model in RAM bool check_tensors; // validate model tensor data bool repack_tensors;// repack if available + bool use_thp; // uase transparent huge pages (linux only) }; // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations diff --git a/src/llama.cpp b/src/llama.cpp index 54e0bb01..3faa3bbe 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -1836,7 +1836,7 @@ struct llama_mmap { // list of mapped fragments (first_offset, last_offset) std::vector> mapped_fragments; - llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */, bool numa = false) { + llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */, bool numa = false, [[maybe_unused]] bool use_thp = false) { size = file->size; int fd = fileno(file->fp); int flags = MAP_SHARED; @@ -1849,6 +1849,28 @@ struct llama_mmap { strerror(errno)); } if (prefetch) { flags |= MAP_POPULATE; } + if (use_thp) { + size_t huge = get_default_huge_page_size(); + auto size = huge*((file->size + huge - 1)/huge); + addr = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); + if (addr != MAP_FAILED) { + printf("%s: using THP with page size %zu MiB ", __func__, huge/(1024*1024)); + fflush(stdout); + size_t tot = 0; + while (tot < file->size) { + auto n_read = pread(fd, static_cast(addr) + tot, file->size - tot, tot); + if (n_read < 0) throw std::runtime_error(format("Reading into mapped huge pages failed at %zu (%s)", tot, strerror(errno))); + printf("."); fflush(stdout); + tot += n_read; + } + printf("\n"); + mapped_fragments.emplace_back(0, file->size); + return; + } + else { + fprintf(stderr, "%s: mmap with huge page size %zu MiB failed (%s)\n", __func__, huge/(1024*1024), strerror(errno)); + } + } #endif addr = mmap(NULL, file->size, PROT_READ, flags, fd, 0); if (addr == MAP_FAILED) { // NOLINT @@ -1935,6 +1957,28 @@ struct llama_mmap { mapped_fragments = std::move(new_mapped_fragments); } +#ifdef __linux__ + static int get_default_huge_page_size() { + int pg_size = 2048; + std::ifstream in("/proc/meminfo"); + if (in) { + std::string line; + while (true) { + std::getline(in, line); + if (in.fail()) break; + if (auto pos = line.find("Hugepagesize:"); pos != std::string::npos) { + std::istringstream str(line.data() + pos + 13); + int aux; + str >> aux; + if (!str.fail()) pg_size = aux; + break; + } + } + } + return pg_size * 1024; + } +#endif + ~llama_mmap() { for (const auto & frag : mapped_fragments) { if (munmap((char *) addr + frag.first, frag.second - frag.first)) { @@ -1945,7 +1989,7 @@ struct llama_mmap { #elif defined(_WIN32) static constexpr bool SUPPORTED = true; - llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1, bool numa = false) { + llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1, bool numa = false, [[maybe_unused]] bool use_thp = false) { GGML_UNUSED(numa); size = file->size; @@ -2007,10 +2051,11 @@ struct llama_mmap { #else static constexpr bool SUPPORTED = false; - llama_mmap(struct llama_file * file, size_t prefetch = -1, bool numa = false) { + llama_mmap(struct llama_file * file, size_t prefetch = -1, bool numa = false, bool use_thp = false) { GGML_UNUSED(file); GGML_UNUSED(prefetch); GGML_UNUSED(numa); + GGML_UNUSED(use_thp); throw std::runtime_error("mmap not supported"); } @@ -3842,6 +3887,7 @@ struct llama_model_loader { bool use_mmap = false; bool check_tensors; bool repack_tensors = false; + bool use_thp = false; llama_files files; llama_ftype ftype; @@ -3876,7 +3922,7 @@ struct llama_model_loader { std::string arch_name; LLM_KV llm_kv = LLM_KV(LLM_ARCH_UNKNOWN); - llama_model_loader(const std::string & fname, bool use_mmap, bool check_tensors, bool repack_tensors, + llama_model_loader(const std::string & fname, bool use_mmap, bool check_tensors, bool repack_tensors, bool use_thp, const llama_model_kv_override * param_overrides_p, const llama_model_tensor_buft_override * param_tensor_buft_overrides_p) { int trace = 0; @@ -4140,6 +4186,7 @@ struct llama_model_loader { this->use_mmap = use_mmap; this->check_tensors = check_tensors; this->repack_tensors = repack_tensors; + this->use_thp = use_thp; } ~llama_model_loader() { @@ -4453,12 +4500,12 @@ struct llama_model_loader { } } - void init_mappings(bool prefetch = true, llama_mlocks * mlock_mmaps = nullptr) { + void init_mappings(bool prefetch = true, llama_mlocks * mlock_mmaps = nullptr, bool use_thp = false) { if (use_mmap) { mappings.reserve(files.size()); mmaps_used.reserve(files.size()); for (const auto & file : files) { - std::unique_ptr mapping(new llama_mmap(file.get(), prefetch ? -1 : 0, ggml_is_numa())); + std::unique_ptr mapping(new llama_mmap(file.get(), prefetch ? -1 : 0, ggml_is_numa(), use_thp)); mmaps_used.emplace_back(mapping->size, 0); if (mlock_mmaps) { std::unique_ptr mlock_mmap(new llama_mlock()); @@ -8077,7 +8124,7 @@ static bool llm_load_tensors( ml.done_getting_tensors(); - ml.init_mappings(true, use_mlock ? &model.mlock_mmaps : nullptr); + ml.init_mappings(true, use_mlock ? &model.mlock_mmaps : nullptr, ml.use_thp); model.mappings.reserve(ml.mappings.size()); // create the backend buffers @@ -8410,7 +8457,7 @@ static bool llm_load_tensors( static int llama_model_load(const std::string & fname, llama_model & model, llama_model_params & params) { try { llama_model_loader ml(fname, params.use_mmap, params.check_tensors, - params.repack_tensors, params.kv_overrides, params.tensor_buft_overrides); + params.repack_tensors, params.use_thp, params.kv_overrides, params.tensor_buft_overrides); model.hparams.vocab_only = params.vocab_only; @@ -17494,7 +17541,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s auto v = (std::vector*)params->kv_overrides; kv_overrides = v->data(); } - llama_model_loader ml(fname_inp, use_mmap, /*check_tensors*/ true, /* repack_tensors */ false, kv_overrides, nullptr); + llama_model_loader ml(fname_inp, use_mmap, /*check_tensors*/ true, /* repack_tensors */ false, /* use_thp */ false, kv_overrides, nullptr); ml.init_mappings(false); // no prefetching llama_model model; @@ -18318,6 +18365,7 @@ struct llama_model_params llama_model_default_params() { /*.use_mlock =*/ false, /*.check_tensors =*/ false, /*.repack_tensors =*/ false, + /*.use_thp =*/ false, }; #ifdef GGML_USE_METAL