microsoft/mscclpp
1
0
Fork 0
mirror of https://github.com/microsoft/mscclpp.git synced 2026-05-12 01:10:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Some cleanup

Browse Source
This commit is contained in:
Changho Hwang
2023-04-11 08:45:22 +00:00
parent 69b5bdfd13
commit d2c2ae72a7
15 changed files with 16 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@@ -118,7 +118,7 @@ endif
LDFLAGS := $(NVLDFLAGS) $(GDRCOPY_LDFLAGS) -libverbs -lnuma
LIBSRCS := $(addprefix src/,debug.cc utils.cc param.cc init.cc proxy.cc ib.cc config.cc)
LIBSRCS := $(addprefix src/,debug.cc utils.cc init.cc proxy.cc ib.cc config.cc)
LIBSRCS += $(addprefix src/bootstrap/,bootstrap.cc socket.cc)
ifneq ($(NPKIT), 0)
LIBSRCS += $(addprefix src/misc/,npkit.cc)

1
src/bootstrap/bootstrap.cc
View File

@@ -6,7 +6,6 @@
#include "bootstrap.h"
#include "config.h"
#include "core.h"
#include "mscclpp.h"
#include "utils.h"
#include <sys/types.h>

7
src/debug.cc
View File

@@ -5,10 +5,11 @@
************************************************************************/
#include "debug.h"
#include "core.h"
#include <cuda_runtime.h>
#include <stdarg.h>
#include <stdlib.h>
#include <sys/syscall.h>
#include <unistd.h>
int mscclppDebugLevel = -1;
static int pid = -1;
@@ -229,15 +230,11 @@ mscclppResult_t mscclppDebugSetLogHandler(mscclppLogHandler_t handler)
return mscclppSuccess;
}
MSCCLPP_PARAM(SetThreadName, "SET_THREAD_NAME", 0);
void mscclppSetThreadName(pthread_t thread, const char* fmt, ...)
{
// pthread_setname_np is nonstandard GNU extension
// needs the following feature test macro
#ifdef _GNU_SOURCE
if (mscclppParamSetThreadName() != 1)
return;
char threadName[MSCCLPP_THREAD_NAMELEN];
va_list vargs;
va_start(vargs, fmt);

1
src/include/alloc.h
View File

@@ -12,7 +12,6 @@
#include "mscclpp.h"
#include "utils.h"
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <unistd.h>

11
src/include/comm.h
View File

@@ -10,16 +10,9 @@
#include "ib.h"
#include "proxy.h"
#if defined(ENABLE_NPKIT)
#include <vector>
// #define CACHE_LINE_SIZE 128
// #define MEM_ALIGN 4096
// #define CUDA_IPC_MIN 2097152UL
// // Channels / LL tuning
// #define MSCCLPP_LL_THREAD_THRESHOLD 8
// #define MSCCLPP_LL128_THREAD_THRESHOLD 8
// #define MSCCLPP_SIMPLE_THREAD_THRESHOLD 64
#endif
#define MAXCONNECTIONS 64

30
src/include/core.h
View File

@@ -1,30 +0,0 @@
/*************************************************************************
* Copyright (c) 2015-2021, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef MSCCLPP_CORE_H_
#define MSCCLPP_CORE_H_
#include "alloc.h"
#include "debug.h"
#include "mscclpp.h"
#include "param.h"
#include <algorithm> // For std::min/std::max
#include <pthread.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#ifdef PROFAPI
#define MSCCLPP_API(ret, func, args...) \
__attribute__((visibility("default"))) __attribute__((alias(#func))) ret p##func(args); \
extern "C" __attribute__((visibility("default"))) __attribute__((weak)) ret func(args)
#else
#define MSCCLPP_API(ret, func, args...) extern "C" __attribute__((visibility("default"))) ret func(args)
#endif // end PROFAPI
#endif // end include guard

4
src/include/mscclpp.h
View File

@@ -12,7 +12,6 @@
#define MSCCLPP_PROXY_FIFO_FLUSH_COUNTER 4
#include <mscclppfifo.h>
#include <time.h>
#include <vector>
#ifdef __cplusplus
@@ -180,7 +179,8 @@ struct mscclppDevConn : mscclppBaseConn
*(volatile uint64_t*)&(localSignalEpochId->device) += 1;
}
#endif
#endif // __CUDACC__
// this is a concurrent fifo which is multiple threads from the device
// can produce for and the sole proxy thread consumes it.
struct mscclppConcurrentFifo fifo;

2
src/include/mscclppfifo.h
View File

@@ -49,7 +49,7 @@ typedef mscclppTrigger* mscclppTrigger_t;
* push() function increments triggerFifoHead, proxyState->fifoTailHost is updated in proxy.cc:mscclppProxyService
* and it occasionally flushes it to triggerFifoTail via a cudaMemcpyAsync.
*
* Why douplicating the tail is a good idea? The fifo is large engouh and we do not need frequent updates
* Why duplicating the tail is a good idea? The fifo is large engouh and we do not need frequent updates
* for the tail as there is usually enough space for device threads to push their work into.
*/
struct mscclppConcurrentFifo

3
src/include/npkit/npkit.h
View File

@@ -2,9 +2,6 @@
#define NPKIT_H_
#include <string>
#include <thread>
#include <cuda_runtime.h>
#include "npkit/npkit_event.h"
#include "npkit/npkit_struct.h"

30
src/include/param.h
View File

@@ -1,30 +0,0 @@
/*************************************************************************
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#ifndef MSCCLPP_PARAM_H_
#define MSCCLPP_PARAM_H_
#include <stdint.h>
const char* userHomeDir();
void setEnvFile(const char* fileName);
void initEnv();
void mscclppLoadParam(char const* env, int64_t deftVal, int64_t uninitialized, int64_t* cache);
#define MSCCLPP_PARAM(name, env, deftVal) \
int64_t mscclppParam##name() \
{ \
constexpr int64_t uninitialized = INT64_MIN; \
static_assert(deftVal != uninitialized, "default value cannot be the uninitialized value."); \
static int64_t cache = uninitialized; \
if (__builtin_expect(__atomic_load_n(&cache, __ATOMIC_RELAXED) == uninitialized, false)) { \
mscclppLoadParam("MSCCLPP_" env, deftVal, uninitialized, &cache); \
} \
return cache; \
}
#endif

6
src/include/utils.h
View File

@@ -8,13 +8,9 @@
#define MSCCLPP_UTILS_H_
#include "alloc.h"
#include "checks.h"
#include "mscclpp.h"
#include <new>
#include <numa.h>
#include <sched.h>
#include <chrono>
#include <stdint.h>
#include <time.h>
// int mscclppCudaCompCap();

7
src/init.cc
View File

@@ -1,6 +1,7 @@
#include "alloc.h"
#include "bootstrap.h"
#include "checks.h"
#include "config.h"
#include "core.h"
#if defined(MSCCLPP_USE_GDRCOPY)
#include "gdr.h"
#endif
@@ -11,6 +12,8 @@
#include "npkit/npkit.h"
#endif
#define MSCCLPP_API(ret, func, args...) extern "C" __attribute__((visibility("default"))) ret func(args)
static uint64_t hashUniqueId(mscclppUniqueId const& id)
{
char const* bytes = (char const*)&id;
@@ -693,4 +696,4 @@ mscclppResult_t mscclppSetBootstrapConnTimeout(int timeout)
mscclppConfig* config = mscclppConfig::getInstance();
config->setBootstrapConnectionTimeoutConfig(timeout);
return mscclppSuccess;
}
}

1
src/misc/npkit.cc
View File

@@ -4,6 +4,7 @@
#include "alloc.h"
#include "npkit/npkit.h"
#include <cuda_runtime.h>
uint64_t NpKit::rank_ = 0;

90
src/param.cc
View File

@@ -1,90 +0,0 @@
/*************************************************************************
* Copyright (c) 2019-2022, NVIDIA CORPORATION. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include "param.h"
#include "debug.h"
#include <algorithm>
#include <errno.h>
#include <pthread.h>
#include <pwd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <unistd.h>
const char* userHomeDir()
{
struct passwd* pwUser = getpwuid(getuid());
return pwUser == NULL ? NULL : pwUser->pw_dir;
}
void setEnvFile(const char* fileName)
{
FILE* file = fopen(fileName, "r");
if (file == NULL)
return;
char* line = NULL;
char envVar[1024];
char envValue[1024];
size_t n = 0;
ssize_t read;
while ((read = getline(&line, &n, file)) != -1) {
if (line[read - 1] == '\n')
line[read - 1] = '\0';
int s = 0; // Env Var Size
while (line[s] != '\0' && line[s] != '=')
s++;
if (line[s] == '\0')
continue;
strncpy(envVar, line, std::min(1023, s));
envVar[s] = '\0';
s++;
strncpy(envValue, line + s, 1023);
envValue[1023] = '\0';
setenv(envVar, envValue, 0);
// printf("%s : %s->%s\n", fileName, envVar, envValue);
}
if (line)
free(line);
fclose(file);
}
void initEnv()
{
char confFilePath[1024];
const char* userDir = userHomeDir();
if (userDir) {
sprintf(confFilePath, "%s/.mscclpp.conf", userDir);
setEnvFile(confFilePath);
}
sprintf(confFilePath, "/etc/mscclpp.conf");
setEnvFile(confFilePath);
}
void mscclppLoadParam(char const* env, int64_t deftVal, int64_t uninitialized, int64_t* cache)
{
static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_lock(&mutex);
if (__atomic_load_n(cache, __ATOMIC_RELAXED) == uninitialized) {
char* str = getenv(env);
int64_t value = deftVal;
if (str && strlen(str) > 0) {
errno = 0;
value = strtoll(str, nullptr, 0);
if (errno) {
value = deftVal;
INFO(MSCCLPP_ALL, "Invalid value %s for %s, using default %lld.", str, env, (long long)deftVal);
} else {
INFO(MSCCLPP_ALL, "%s set by environment to %lld.", env, (long long)value);
}
}
__atomic_store_n(cache, value, __ATOMIC_RELAXED);
}
pthread_mutex_unlock(&mutex);
}

2
src/utils.cc
View File

@@ -5,8 +5,8 @@
************************************************************************/
#include "utils.h"
#include "core.h"
#include <numa.h>
#include <stdlib.h>
#include <string>