From d2c2ae72a7ef2b65e68738c38d8a3de424aa3f65 Mon Sep 17 00:00:00 2001 From: Changho Hwang Date: Tue, 11 Apr 2023 08:45:22 +0000 Subject: [PATCH] Some cleanup --- Makefile | 2 +- src/bootstrap/bootstrap.cc | 1 - src/debug.cc | 7 +-- src/include/alloc.h | 1 - src/include/comm.h | 11 +---- src/include/core.h | 30 ------------- src/include/mscclpp.h | 4 +- src/include/mscclppfifo.h | 2 +- src/include/npkit/npkit.h | 3 -- src/include/param.h | 30 ------------- src/include/utils.h | 6 +-- src/init.cc | 7 ++- src/misc/npkit.cc | 1 + src/param.cc | 90 -------------------------------------- src/utils.cc | 2 +- 15 files changed, 16 insertions(+), 181 deletions(-) delete mode 100644 src/include/core.h delete mode 100644 src/include/param.h delete mode 100644 src/param.cc diff --git a/Makefile b/Makefile index 14f7bc81..99f2fcd6 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,7 @@ endif LDFLAGS := $(NVLDFLAGS) $(GDRCOPY_LDFLAGS) -libverbs -lnuma -LIBSRCS := $(addprefix src/,debug.cc utils.cc param.cc init.cc proxy.cc ib.cc config.cc) +LIBSRCS := $(addprefix src/,debug.cc utils.cc init.cc proxy.cc ib.cc config.cc) LIBSRCS += $(addprefix src/bootstrap/,bootstrap.cc socket.cc) ifneq ($(NPKIT), 0) LIBSRCS += $(addprefix src/misc/,npkit.cc) diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 064af4a8..11389222 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -6,7 +6,6 @@ #include "bootstrap.h" #include "config.h" -#include "core.h" #include "mscclpp.h" #include "utils.h" #include diff --git a/src/debug.cc b/src/debug.cc index a3807d3e..d6d29262 100644 --- a/src/debug.cc +++ b/src/debug.cc @@ -5,10 +5,11 @@ ************************************************************************/ #include "debug.h" -#include "core.h" +#include #include #include #include +#include int mscclppDebugLevel = -1; static int pid = -1; @@ -229,15 +230,11 @@ mscclppResult_t mscclppDebugSetLogHandler(mscclppLogHandler_t handler) return mscclppSuccess; } -MSCCLPP_PARAM(SetThreadName, "SET_THREAD_NAME", 0); - void mscclppSetThreadName(pthread_t thread, const char* fmt, ...) { // pthread_setname_np is nonstandard GNU extension // needs the following feature test macro #ifdef _GNU_SOURCE - if (mscclppParamSetThreadName() != 1) - return; char threadName[MSCCLPP_THREAD_NAMELEN]; va_list vargs; va_start(vargs, fmt); diff --git a/src/include/alloc.h b/src/include/alloc.h index 496af197..5c696e6e 100644 --- a/src/include/alloc.h +++ b/src/include/alloc.h @@ -12,7 +12,6 @@ #include "mscclpp.h" #include "utils.h" #include -#include #include #include diff --git a/src/include/comm.h b/src/include/comm.h index 28f3cb0d..38abd438 100644 --- a/src/include/comm.h +++ b/src/include/comm.h @@ -10,16 +10,9 @@ #include "ib.h" #include "proxy.h" +#if defined(ENABLE_NPKIT) #include - -// #define CACHE_LINE_SIZE 128 -// #define MEM_ALIGN 4096 -// #define CUDA_IPC_MIN 2097152UL - -// // Channels / LL tuning -// #define MSCCLPP_LL_THREAD_THRESHOLD 8 -// #define MSCCLPP_LL128_THREAD_THRESHOLD 8 -// #define MSCCLPP_SIMPLE_THREAD_THRESHOLD 64 +#endif #define MAXCONNECTIONS 64 diff --git a/src/include/core.h b/src/include/core.h deleted file mode 100644 index e3213bd6..00000000 --- a/src/include/core.h +++ /dev/null @@ -1,30 +0,0 @@ -/************************************************************************* - * Copyright (c) 2015-2021, NVIDIA CORPORATION. All rights reserved. - * - * See LICENSE.txt for license information - ************************************************************************/ - -#ifndef MSCCLPP_CORE_H_ -#define MSCCLPP_CORE_H_ - -#include "alloc.h" -#include "debug.h" -#include "mscclpp.h" -#include "param.h" -#include // For std::min/std::max -#include -#include -#include -#include -#include -#include - -#ifdef PROFAPI -#define MSCCLPP_API(ret, func, args...) \ - __attribute__((visibility("default"))) __attribute__((alias(#func))) ret p##func(args); \ - extern "C" __attribute__((visibility("default"))) __attribute__((weak)) ret func(args) -#else -#define MSCCLPP_API(ret, func, args...) extern "C" __attribute__((visibility("default"))) ret func(args) -#endif // end PROFAPI - -#endif // end include guard diff --git a/src/include/mscclpp.h b/src/include/mscclpp.h index 33ddc47b..c67add94 100644 --- a/src/include/mscclpp.h +++ b/src/include/mscclpp.h @@ -12,7 +12,6 @@ #define MSCCLPP_PROXY_FIFO_FLUSH_COUNTER 4 #include -#include #include #ifdef __cplusplus @@ -180,7 +179,8 @@ struct mscclppDevConn : mscclppBaseConn *(volatile uint64_t*)&(localSignalEpochId->device) += 1; } -#endif +#endif // __CUDACC__ + // this is a concurrent fifo which is multiple threads from the device // can produce for and the sole proxy thread consumes it. struct mscclppConcurrentFifo fifo; diff --git a/src/include/mscclppfifo.h b/src/include/mscclppfifo.h index 78918fff..341025b5 100644 --- a/src/include/mscclppfifo.h +++ b/src/include/mscclppfifo.h @@ -49,7 +49,7 @@ typedef mscclppTrigger* mscclppTrigger_t; * push() function increments triggerFifoHead, proxyState->fifoTailHost is updated in proxy.cc:mscclppProxyService * and it occasionally flushes it to triggerFifoTail via a cudaMemcpyAsync. * - * Why douplicating the tail is a good idea? The fifo is large engouh and we do not need frequent updates + * Why duplicating the tail is a good idea? The fifo is large engouh and we do not need frequent updates * for the tail as there is usually enough space for device threads to push their work into. */ struct mscclppConcurrentFifo diff --git a/src/include/npkit/npkit.h b/src/include/npkit/npkit.h index a0691afd..f0a72dfc 100644 --- a/src/include/npkit/npkit.h +++ b/src/include/npkit/npkit.h @@ -2,9 +2,6 @@ #define NPKIT_H_ #include -#include - -#include #include "npkit/npkit_event.h" #include "npkit/npkit_struct.h" diff --git a/src/include/param.h b/src/include/param.h deleted file mode 100644 index e7478807..00000000 --- a/src/include/param.h +++ /dev/null @@ -1,30 +0,0 @@ -/************************************************************************* - * Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved. - * - * See LICENSE.txt for license information - ************************************************************************/ - -#ifndef MSCCLPP_PARAM_H_ -#define MSCCLPP_PARAM_H_ - -#include - -const char* userHomeDir(); -void setEnvFile(const char* fileName); -void initEnv(); - -void mscclppLoadParam(char const* env, int64_t deftVal, int64_t uninitialized, int64_t* cache); - -#define MSCCLPP_PARAM(name, env, deftVal) \ - int64_t mscclppParam##name() \ - { \ - constexpr int64_t uninitialized = INT64_MIN; \ - static_assert(deftVal != uninitialized, "default value cannot be the uninitialized value."); \ - static int64_t cache = uninitialized; \ - if (__builtin_expect(__atomic_load_n(&cache, __ATOMIC_RELAXED) == uninitialized, false)) { \ - mscclppLoadParam("MSCCLPP_" env, deftVal, uninitialized, &cache); \ - } \ - return cache; \ - } - -#endif diff --git a/src/include/utils.h b/src/include/utils.h index 3eff9842..59b35407 100644 --- a/src/include/utils.h +++ b/src/include/utils.h @@ -8,13 +8,9 @@ #define MSCCLPP_UTILS_H_ #include "alloc.h" -#include "checks.h" #include "mscclpp.h" -#include -#include -#include +#include #include -#include // int mscclppCudaCompCap(); diff --git a/src/init.cc b/src/init.cc index 803f1916..db6ee645 100644 --- a/src/init.cc +++ b/src/init.cc @@ -1,6 +1,7 @@ +#include "alloc.h" #include "bootstrap.h" +#include "checks.h" #include "config.h" -#include "core.h" #if defined(MSCCLPP_USE_GDRCOPY) #include "gdr.h" #endif @@ -11,6 +12,8 @@ #include "npkit/npkit.h" #endif +#define MSCCLPP_API(ret, func, args...) extern "C" __attribute__((visibility("default"))) ret func(args) + static uint64_t hashUniqueId(mscclppUniqueId const& id) { char const* bytes = (char const*)&id; @@ -693,4 +696,4 @@ mscclppResult_t mscclppSetBootstrapConnTimeout(int timeout) mscclppConfig* config = mscclppConfig::getInstance(); config->setBootstrapConnectionTimeoutConfig(timeout); return mscclppSuccess; -} \ No newline at end of file +} diff --git a/src/misc/npkit.cc b/src/misc/npkit.cc index 4a7eb849..30914810 100644 --- a/src/misc/npkit.cc +++ b/src/misc/npkit.cc @@ -4,6 +4,7 @@ #include "alloc.h" #include "npkit/npkit.h" +#include uint64_t NpKit::rank_ = 0; diff --git a/src/param.cc b/src/param.cc deleted file mode 100644 index 2af48084..00000000 --- a/src/param.cc +++ /dev/null @@ -1,90 +0,0 @@ -/************************************************************************* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. All rights reserved. - * - * See LICENSE.txt for license information - ************************************************************************/ - -#include "param.h" -#include "debug.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -const char* userHomeDir() -{ - struct passwd* pwUser = getpwuid(getuid()); - return pwUser == NULL ? NULL : pwUser->pw_dir; -} - -void setEnvFile(const char* fileName) -{ - FILE* file = fopen(fileName, "r"); - if (file == NULL) - return; - - char* line = NULL; - char envVar[1024]; - char envValue[1024]; - size_t n = 0; - ssize_t read; - while ((read = getline(&line, &n, file)) != -1) { - if (line[read - 1] == '\n') - line[read - 1] = '\0'; - int s = 0; // Env Var Size - while (line[s] != '\0' && line[s] != '=') - s++; - if (line[s] == '\0') - continue; - strncpy(envVar, line, std::min(1023, s)); - envVar[s] = '\0'; - s++; - strncpy(envValue, line + s, 1023); - envValue[1023] = '\0'; - setenv(envVar, envValue, 0); - // printf("%s : %s->%s\n", fileName, envVar, envValue); - } - if (line) - free(line); - fclose(file); -} - -void initEnv() -{ - char confFilePath[1024]; - const char* userDir = userHomeDir(); - if (userDir) { - sprintf(confFilePath, "%s/.mscclpp.conf", userDir); - setEnvFile(confFilePath); - } - sprintf(confFilePath, "/etc/mscclpp.conf"); - setEnvFile(confFilePath); -} - -void mscclppLoadParam(char const* env, int64_t deftVal, int64_t uninitialized, int64_t* cache) -{ - static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; - pthread_mutex_lock(&mutex); - if (__atomic_load_n(cache, __ATOMIC_RELAXED) == uninitialized) { - char* str = getenv(env); - int64_t value = deftVal; - if (str && strlen(str) > 0) { - errno = 0; - value = strtoll(str, nullptr, 0); - if (errno) { - value = deftVal; - INFO(MSCCLPP_ALL, "Invalid value %s for %s, using default %lld.", str, env, (long long)deftVal); - } else { - INFO(MSCCLPP_ALL, "%s set by environment to %lld.", env, (long long)value); - } - } - __atomic_store_n(cache, value, __ATOMIC_RELAXED); - } - pthread_mutex_unlock(&mutex); -} diff --git a/src/utils.cc b/src/utils.cc index c0766765..ebd31bfe 100644 --- a/src/utils.cc +++ b/src/utils.cc @@ -5,8 +5,8 @@ ************************************************************************/ #include "utils.h" -#include "core.h" +#include #include #include