Make IB more configurable (#703)

* Added `port` and `gidIndex` field in the IB endpoint config (and
`deviceIndex` field for future usages)
* Added `MSCCLPP_IBV_SO` env variable to specify a custom libibverbs.so
* Added `--ib_gid_index` CLI option to `mp_unit_tests`
* Other minor fixes
This commit is contained in:
Changho Hwang
2025-12-18 13:21:07 -08:00
committed by GitHub
parent 11b7b35832
commit 9e076da3d4
14 changed files with 182 additions and 84 deletions

View File

@@ -126,10 +126,16 @@ void register_core(nb::module_& m) {
nb::class_<EndpointConfig::Ib>(m, "EndpointConfigIb")
.def(nb::init<>())
.def(nb::init<int, int, int, int>(), nb::arg("max_cq_size") = EndpointConfig::Ib::DefaultMaxCqSize,
.def(nb::init<int, int, int, int, int, int, int>(), nb::arg("device_index") = -1,
nb::arg("port") = EndpointConfig::Ib::DefaultPort,
nb::arg("gid_index") = EndpointConfig::Ib::DefaultGidIndex,
nb::arg("max_cq_size") = EndpointConfig::Ib::DefaultMaxCqSize,
nb::arg("max_cq_poll_num") = EndpointConfig::Ib::DefaultMaxCqPollNum,
nb::arg("max_send_wr") = EndpointConfig::Ib::DefaultMaxSendWr,
nb::arg("max_wr_per_send") = EndpointConfig::Ib::DefaultMaxWrPerSend)
.def_rw("device_index", &EndpointConfig::Ib::deviceIndex)
.def_rw("port", &EndpointConfig::Ib::port)
.def_rw("gid_index", &EndpointConfig::Ib::gidIndex)
.def_rw("max_cq_size", &EndpointConfig::Ib::maxCqSize)
.def_rw("max_cq_poll_num", &EndpointConfig::Ib::maxCqPollNum)
.def_rw("max_send_wr", &EndpointConfig::Ib::maxSendWr)
@@ -176,6 +182,15 @@ void register_core(nb::module_& m) {
.def_rw("transport", &EndpointConfig::transport)
.def_rw("device", &EndpointConfig::device)
.def_rw("ib", &EndpointConfig::ib)
.def_prop_rw(
"ib_device_index", [](EndpointConfig& self) { return self.ib.deviceIndex; },
[](EndpointConfig& self, int v) { self.ib.deviceIndex = v; })
.def_prop_rw(
"ib_port", [](EndpointConfig& self) { return self.ib.port; },
[](EndpointConfig& self, int v) { self.ib.port = v; })
.def_prop_rw(
"ib_gid_index", [](EndpointConfig& self) { return self.ib.gidIndex; },
[](EndpointConfig& self, int v) { self.ib.gidIndex = v; })
.def_prop_rw(
"ib_max_cq_size", [](EndpointConfig& self) { return self.ib.maxCqSize; },
[](EndpointConfig& self, int v) { self.ib.maxCqSize = v; })